diff --git a/README.md b/README.md index 5d5e6986d..fdc20cb6f 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,7 @@ const metascraper = require('metascraper')([ - [metascraper-logo](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-logo) – Get logo property from HTML markup. - [metascraper-manifest](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-manifest) – Metascraper integration for detecting PWA Web app [manifests](https://developer.mozilla.org/en-US/docs/Web/Manifest). - [metascraper-media-provider](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-media-provider) – Get specific video provider url (Facebook/Twitter/Vimeo/etc). +- [metascraper-pdf](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-pdf) – Get title, author, date, description, publisher, image, logo, and lang from a PDF document. - [metascraper-publisher](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-publisher) – Get publisher property from HTML markup. - [metascraper-readability](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-readability) – A Mozilla readability connector for metascraper. - [metascraper-title](https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title) – Get title property from HTML markup. diff --git a/packages/metascraper-pdf/README.md b/packages/metascraper-pdf/README.md new file mode 100644 index 000000000..43bd7382d --- /dev/null +++ b/packages/metascraper-pdf/README.md @@ -0,0 +1,124 @@ +
+
+ metascraper +
+
+

metascraper-pdf: Get title, author, date, description, publisher, image, logo, and lang out of a PDF document.

+

See our website for more information.

+
+
+ +## Install + +```bash +$ npm install metascraper-pdf --save +``` + +## Usage + +The rules download the document at `url` when it looks like a PDF. + +```js +const metascraper = require('metascraper')([require('metascraper-pdf')()]) + +const metadata = await metascraper({ + url: 'https://arxiv.org/pdf/1706.03762v7' +}) + +// { +// title: 'Attention Is All You Need', +// author: 'Ashish Vaswani', +// publisher: 'arXiv', +// date: '2017-06-01T00:00:00.000Z', +// description: 'The dominant sequence transduction models are based on…', +// lang: 'en', +// logo: 'https://www.google.com/s2/favicons?domain_url=…', +// image: null +// } +``` + +The bundle is a no-op unless [`.test()`](#testprops) sees a PDF URL, so it is safe to mix with the HTML rules: + +```js +const metascraper = require('metascraper')([ + require('metascraper-pdf')(), + require('metascraper-title')(), + require('metascraper-author')() +]) + +const metadata = await metascraper({ url, html }) +``` + +## How it reads a document + +The package fetches the URL, then reads the page the way a person does. Embedded PDF metadata is +mostly unusable — arXiv ships an empty `Title`, LaTeX ships `pedregosa11a.dvi`, Word ships +`Microsoft Word - draft.docx`, conference templates ship the venue as the `Subject`. + +- **title** — the largest type on the first page, skipping the banner publishers print above it + (`NBER WORKING PAPER SERIES`, `arXiv:2303.08774v6`, `REVIEW`) and any byline set in the same size. +- **author** — the block under the title, stripped of emails, affiliation superscripts and + organisation names. Following metascraper's convention this returns a single name. +- **description** — the abstract, or the first paragraph of body text when there is no abstract. +- **publisher** — the venue in the running header or footer; otherwise a known host (`arXiv`, + `NBER`, `PLOS`) or the domain. +- **date** — the identifier when the url encodes it (an arXiv id is a year-month; proceedings hosts + put the year in the path). Otherwise the markers on the page win over the PDF creation date. +- **lang** — `dc:language` when present, then the host, then the words on the page. +- **image** — a first-page figure encoded as a PNG data URI, when the PDF embeds one that is not + just a decoration. +- **logo** — a first-page mark encoded as a PNG data URI, or the publisher favicon. + +## API + +### metascraper-pdf([options]) + +#### options + +##### maxPages + +Type: `number`
+Default: `2` + +How many pages to read text from. The title, author and publisher only ever come from the first +page; the extra page feeds the description when a document has no abstract. + +##### gotOpts + +Type: `object` + +Any option provided here will be passed to [got#options](https://github.com/sindresorhus/got#options). + +##### keyvOpts + +Type: `object` + +Any option provided here will be passed to [@keyvhq/memoize#options](https://github.com/microlinkhq/keyv/tree/master/packages/memoize#keyvoptions). + +##### getPdf + +Type: `function` + +It will be called to get the PDF bytes behind `url`. Defaults to downloading the URL with `got`. + +### .test(props) + +Type: `function`
+Returns: `boolean` + +`true` when `props.url` points at a PDF (`.pdf`, an `/pdf` path, or `type=printable`), which is how +the bundle stays inert for HTML input. + +```js +const { test: isPdf } = require('metascraper-pdf') + +isPdf({ url: 'https://arxiv.org/pdf/1706.03762v7' }) // => true +isPdf({ url: 'https://example.com' }) // => false +``` + +## License + +**metascraper-pdf** © [microlink.io](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/metascraper/blob/master/LICENSE.md) License.
+Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/microlinkhq/metascraper/contributors). + +> [microlink.io](https://microlink.io) · GitHub [microlink.io](https://github.com/microlinkhq) · X [@microlinkhq](https://x.com/microlinkhq) diff --git a/packages/metascraper-pdf/package.json b/packages/metascraper-pdf/package.json new file mode 100644 index 000000000..ce8e25205 --- /dev/null +++ b/packages/metascraper-pdf/package.json @@ -0,0 +1,58 @@ +{ + "name": "metascraper-pdf", + "description": "Metascraper rules for extracting title, author, date, description, publisher, image, logo, and lang from PDF documents.", + "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-pdf", + "version": "5.57.0", + "types": "src/index.d.ts", + "main": "src/index.js", + "author": { + "email": "hello@microlink.io", + "name": "microlink.io", + "url": "https://microlink.io" + }, + "repository": { + "directory": "packages/metascraper-pdf", + "type": "git", + "url": "git+https://github.com/microlinkhq/metascraper.git" + }, + "bugs": { + "url": "https://github.com/microlinkhq/metascraper/issues" + }, + "keywords": [ + "document", + "extract", + "metadata", + "metascraper", + "paper", + "pdf", + "scraper" + ], + "dependencies": { + "@keyvhq/memoize": "~2.2.4", + "@metascraper/helpers": "workspace:*", + "async-memoize-one": "~1.2.1", + "got": "~11.8.6", + "unpdf": "~1.8.1" + }, + "devDependencies": { + "async-listen": "~3.1.0", + "ava": "8", + "metascraper": "workspace:*" + }, + "engines": { + "node": ">= 22" + }, + "files": [ + "src" + ], + "scripts": { + "test": "NODE_PATH=.. TZ=UTC ava --timeout 30s" + }, + "license": "MIT", + "ava": { + "files": [ + "test/**/*.js", + "!test/helpers/**" + ] + } +} diff --git a/packages/metascraper-pdf/src/author.js b/packages/metascraper-pdf/src/author.js new file mode 100644 index 000000000..7e60902e6 --- /dev/null +++ b/packages/metascraper-pdf/src/author.js @@ -0,0 +1,153 @@ +'use strict' + +const { + ORGANIZATION_WORDS, + PLACE_NAME, + flatten, + isBannerLine, + isInvertedName, + isPersonName, + splitNamePairs, + splitNames, + stripNoise, + tidy +} = require('./text') + +const EDITOR_PREFIX = + /^(edited|reviewed|approved|submitted|received|accepted|published)\s+by\s*:?|^(editors?|reviewing editors?|action editors?)\s*:/i +const SECTION_WORDS = + /^(abstract|summary|introduction|contents|table of contents|keywords|index|preface|foreword|version|draft)\b/i + +const AUTHOR_BLOCK_MARGIN = 4 +const EDITOR_BLOCK_LINES = 4 +const MAX_ORGANIZATION_WORDS = 4 +const MAX_AUTHORS = 10 +const NEIGHBOUR_OFFSETS = [1, 2, 3, 4] + +const isCapitalizedWord = word => /^[\p{Lu}]/u.test(word) + +const isOrganizationAuthor = text => { + if (/\S+@\S+/.test(text) || /\d/.test(text) || /^https?:/i.test(text)) { + return false + } + if (SECTION_WORDS.test(text) || PLACE_NAME.test(text)) return false + const words = text.split(/\s+/) + return ( + words.length <= MAX_ORGANIZATION_WORDS && words.every(isCapitalizedWord) + ) +} + +const editorBlock = lines => { + const excluded = new Set() + + for (const line of lines) { + if (!EDITOR_PREFIX.test(line.text)) continue + for (let offset = 0; offset <= EDITOR_BLOCK_LINES; offset++) { + excluded.add(line.index + offset) + } + } + + return excluded +} + +const toAuthor = (lines, indexes, options = {}) => { + const { organizationLimit = Infinity, allowOrganization = false } = options + const excluded = editorBlock(lines) + const usable = index => !excluded.has(index) + + const names = indexes + .filter(usable) + .map(index => lines[index]) + .filter(Boolean) + .map(line => line.text) + .filter( + text => + !EDITOR_PREFIX.test(text) && + !ORGANIZATION_WORDS.test(text) && + !isBannerLine(text) + ) + .flatMap(text => stripNoise(text).split(/;\s*/).flatMap(splitNames)) + .map(tidy) + .filter(isPersonName) + + const unique = [...new Set(names.map(flatten))].slice(0, MAX_AUTHORS) + if (unique.length > 1) return unique.join(', ') + + const paired = indexes + .filter(usable) + .map(index => lines[index]) + .filter(line => line && !ORGANIZATION_WORDS.test(line.text)) + .flatMap(line => splitNamePairs(stripNoise(line.text))) + .filter(isPersonName) + if (paired.length > unique.length) { + return [...new Set(paired)].slice(0, MAX_AUTHORS).join(', ') + } + if (unique.length > 0) return unique.join(', ') + + if (!allowOrganization) return null + + const organization = indexes + .filter(index => index <= organizationLimit && usable(index)) + .map(index => lines[index]) + .filter(Boolean) + .map(line => flatten(stripNoise(line.text))) + .find(isOrganizationAuthor) + + return organization || null +} + +/** + * Bylines share a font size. Once one name is found, every line set in the same + * size around it belongs to the same block, which is what recovers the authors + * hidden between affiliation and email lines. + */ +const expandAuthorLines = (lines, indexes, { titleIndexes = [] } = {}) => { + const excludedTitle = new Set(titleIndexes) + const usable = indexes.filter(index => !excludedTitle.has(index)) + const named = usable + .map(index => lines[index]) + .filter(line => line && isPersonName(stripNoise(line.text))) + + if (named.length === 0) return usable + + const sizes = new Set(named.map(line => line.size)) + const first = Math.min(...named.map(line => line.index)) - AUTHOR_BLOCK_MARGIN + const last = Math.max(...named.map(line => line.index)) + AUTHOR_BLOCK_MARGIN + + return lines + .filter( + line => + line.index >= first && + line.index <= last && + !excludedTitle.has(line.index) + ) + .filter(line => sizes.has(line.size) && isPersonName(stripNoise(line.text))) + .map(line => line.index) +} + +const nameCount = value => { + if (!value) return 0 + return value.split(/\s*;\s*/).reduce((count, part) => { + const trimmed = part.trim() + if (!trimmed) return count + if (isInvertedName(trimmed)) return count + 1 + return count + trimmed.split(/,|\s+and\s+/i).filter(Boolean).length + }, 0) +} + +const getAuthor = (lines, { titleIndexes = [] } = {}) => { + const titleIndex = + titleIndexes.length > 0 ? titleIndexes[titleIndexes.length - 1] : 0 + const neighbours = NEIGHBOUR_OFFSETS.map(offset => titleIndex + offset) + const organization = { + allowOrganization: true, + organizationLimit: titleIndex + AUTHOR_BLOCK_MARGIN + } + + return ( + toAuthor(lines, expandAuthorLines(lines, neighbours, { titleIndexes })) || + toAuthor(lines, neighbours, organization) + ) +} + +module.exports = { expandAuthorLines, getAuthor, nameCount, toAuthor } diff --git a/packages/metascraper-pdf/src/date.js b/packages/metascraper-pdf/src/date.js new file mode 100644 index 000000000..1fd4985a7 --- /dev/null +++ b/packages/metascraper-pdf/src/date.js @@ -0,0 +1,120 @@ +'use strict' + +const YEAR = '((?:19|20)\\d{2})' +const MONTHS = + 'january|february|march|april|may|june|july|august|september|october|november|december|enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre' + +const ARXIV_NEW = + /arxiv\.org\/(?:abs|pdf)\/(\d{2})(\d{2})\.\d{4,5}|arxiv:(\d{2})(\d{2})\.\d{4,5}/i +const ARXIV_OLD = + /arxiv\.org\/(?:abs|pdf)\/[a-z-]+(?:\.[A-Z]{2})?\/(\d{2})(\d{2})\d{3}|arxiv:[a-z-]+(?:\.[A-Z]{2})?\/(\d{2})(\d{2})\d{3}/i +const PATH_YEAR = /(?:^|[/_-])((?:19|20)\d{2})(?:[/_-]|$)/ + +const MARKED_YEAR = [ + new RegExp( + `\\b(?:published|accepted|issued|publicado|publication date)\\b[^\\n]{0,40}?${YEAR}`, + 'i' + ), + new RegExp(`(?:©|\\(c\\)|copyright)[^\\n]{0,40}?${YEAR}`, 'i'), + new RegExp(`\\barxiv:\\S+[^\\n]{0,30}?${YEAR}`, 'i'), + new RegExp(`\\b(?:vol\\.?|volume|núm\\.?|no\\.)[^\\n]{0,40}?${YEAR}\\b`, 'i'), + new RegExp(`\\b(?:${MONTHS})\\s+${YEAR}\\b`, 'i'), + new RegExp( + `\\b(?:cvpr|iccv|eccv|neurips|nips|icml|iclr|acl|emnlp|naacl|interspeech|aaai|ijcai)\\s*${YEAR}\\b`, + 'i' + ), + new RegExp( + `${YEAR}\\s+(?:ieee|acm|international conference|conference on)`, + 'i' + ) +] + +const HEADER_LINES = 12 +const FOOTER_LINES = 4 +const BODY_LINE_WORDS = 18 +const ARXIV_EPOCH = 1991 + +const toFullYear = shortYear => { + const year = Number(shortYear) + return year >= 91 ? 1900 + year : 2000 + year +} + +const arxivStamp = source => { + const match = ARXIV_NEW.exec(source) || ARXIV_OLD.exec(source) + if (!match) return null + const [year, month] = match.slice(1).filter(Boolean) + return { year: toFullYear(year), month: Number(month) } +} + +/** + * An arXiv id encodes the month it was announced, and proceedings hosts put the + * year in the path. Both beat the PDF creation date, which is the day the file + * was last built. + */ +const identifierDate = (url, { text = '' } = {}) => { + const stamp = arxivStamp(url) || arxivStamp(text.slice(0, 400)) + if ( + stamp && + stamp.year >= ARXIV_EPOCH && + stamp.month >= 1 && + stamp.month <= 12 + ) { + const month = String(stamp.month).padStart(2, '0') + return `${stamp.year}-${month}-01T00:00:00.000Z` + } + + const { pathname } = new URL(url) + const match = PATH_YEAR.exec(pathname) + const year = match ? Number(match[1]) : 0 + return year >= 1900 ? `${year}-01-01T00:00:00.000Z` : null +} + +const headerAndFooter = lines => { + const header = [] + + for (const line of lines) { + if (header.length >= HEADER_LINES) break + header.push(line.text) + if (line.text.split(/\s+/).length >= BODY_LINE_WORDS && header.length > 1) { + break + } + } + + return [...header, ...lines.slice(-FOOTER_LINES).map(line => line.text)].join( + '\n' + ) +} + +const firstMarkedYear = (text, currentYear) => { + for (const pattern of MARKED_YEAR) { + const match = pattern.exec(text) + const year = match ? Number(match[1]) : 0 + if (year >= 1900 && year <= currentYear) return year + } + return null +} + +const documentYear = (lines, { embedded = {}, now = new Date() } = {}) => { + const currentYear = now.getUTCFullYear() + const venue = [embedded.description, embedded.keywords, embedded.title] + .filter(Boolean) + .join('\n') + return ( + firstMarkedYear(headerAndFooter(lines), currentYear) || + firstMarkedYear(venue, currentYear) + ) +} + +const getDate = (url, { lines, text, embedded, rawEmbedded, now }) => { + const fromIdentifier = identifierDate(url, { text }) + if (fromIdentifier) return fromIdentifier + + const year = documentYear(lines, { embedded: rawEmbedded, now }) + if (year == null) return embedded.date + if (embedded.date && new Date(embedded.date).getUTCFullYear() === year) { + return embedded.date + } + return `${year}-01-01T00:00:00.000Z` +} + +module.exports = { documentYear, getDate, identifierDate } diff --git a/packages/metascraper-pdf/src/description.js b/packages/metascraper-pdf/src/description.js new file mode 100644 index 000000000..0f3b9d71b --- /dev/null +++ b/packages/metascraper-pdf/src/description.js @@ -0,0 +1,63 @@ +'use strict' + +const ABSTRACT_HEADING = + /(?:^|\n)\s*(?:abstract|summary|executive summary)\b[.:—-]?\s*/i +const NEXT_SECTION = + /(?:^|\n)\s*(?:(?:\d+|[ivxlc]+)\.?\s+)?(?:introduction|keywords|contents|table of contents|references|acknowledgements?)\b/i +const RUNNING_HEADER = /[|·•‖]|^\d+\s*$|\bdoi:/i +const SENTENCE_END = /(?<=[.!?])\s+/ +const MAX_SUMMARY_LENGTH = 300 +const MIN_SUMMARY_LENGTH = 40 + +const dehyphenate = text => text.replace(/(\p{Ll})-\s+(\p{Ll})/gu, '$1$2') + +/** The size that carries most of the page is the body text, not the furniture. */ +const dominantSize = lines => { + const weight = new Map() + for (const line of lines) { + weight.set(line.size, (weight.get(line.size) || 0) + line.text.length) + } + return [...weight.entries()] + .sort((left, right) => right[1] - left[1]) + .map(([size]) => size)[0] +} + +const firstSentences = (text, maxLength) => { + const sentences = text.split(SENTENCE_END) + let summary = '' + + for (const sentence of sentences) { + if (summary && `${summary} ${sentence}`.length > maxLength) break + summary = summary ? `${summary} ${sentence}` : sentence + if (summary.length >= maxLength) break + } + + if (summary.length <= maxLength) return summary.trim() + const clipped = summary.slice(0, maxLength) + return `${clipped.slice(0, clipped.lastIndexOf(' ')).trim()}…` +} + +/** + * The abstract, or the first real paragraph when the document has none. + */ +const getDescription = (lines, { maxLength = MAX_SUMMARY_LENGTH } = {}) => { + const readable = lines.filter(line => !RUNNING_HEADER.test(line.text)) + const text = dehyphenate(readable.map(line => line.text).join('\n')) + const abstract = ABSTRACT_HEADING.exec(text) + + const bodySize = dominantSize(readable) + const body = abstract + ? text.slice(abstract.index + abstract[0].length).split(NEXT_SECTION)[0] + : readable + .filter(line => line.size === bodySize) + .map(line => line.text) + .join(' ') + + const summary = firstSentences( + dehyphenate(body).replace(/\s+/g, ' ').trim(), + maxLength + ) + return summary.length >= MIN_SUMMARY_LENGTH ? summary : null +} + +module.exports = { getDescription } diff --git a/packages/metascraper-pdf/src/document.js b/packages/metascraper-pdf/src/document.js new file mode 100644 index 000000000..8494e3dab --- /dev/null +++ b/packages/metascraper-pdf/src/document.js @@ -0,0 +1,98 @@ +'use strict' + +const { extractImages, getDocumentProxy } = require('unpdf') + +const SAME_LINE_TOLERANCE = 2 +const MIN_LINE_LENGTH = 2 +const WORD_GAP_RATIO = 0.08 +const COLUMN_GAP_RATIO = 1.5 +const DEFAULT_MAX_PAGES = 2 + +const normalizeSpaces = text => + text + .replace(/[^\S ]+/g, ' ') + .replace(/ {3,}/g, ' ') + .trim() + +const toLines = items => { + const lines = [] + + for (const item of items) { + if (!item.str.trim()) continue + const x = item.transform[4] + const y = item.transform[5] + const size = item.height || item.transform[0] + const previous = lines[lines.length - 1] + + if (previous && Math.abs(previous.y - y) <= SAME_LINE_TOLERANCE) { + const gap = x - previous.endX + const separator = gap > size * COLUMN_GAP_RATIO ? ' ' : ' ' + const needsSpace = + gap > size * WORD_GAP_RATIO && + !/\s$/.test(previous.text) && + !/^\s/.test(item.str) + previous.text += needsSpace ? `${separator}${item.str}` : item.str + previous.size = Math.max(previous.size, size) + previous.endX = x + item.width + } else { + lines.push({ y, size, text: item.str, endX: x + item.width }) + } + } + + return lines + .map(({ y, size, text }) => ({ + y, + size: Number(size.toFixed(2)), + text: normalizeSpaces(text) + })) + .filter(line => line.text.length >= MIN_LINE_LENGTH) +} + +const pageLines = async (pdf, pageNumber) => { + const page = await pdf.getPage(pageNumber) + const { items } = await page.getTextContent() + const height = page.view[3] + return toLines(items).map(line => ({ ...line, pageHeight: height })) +} + +const readDocument = async (buffer, { maxPages = DEFAULT_MAX_PAGES } = {}) => { + const pdf = await getDocumentProxy(Uint8Array.from(buffer), { + isEvalSupported: false, + verbosity: 0 + }) + + try { + const { info, metadata } = await pdf.getMetadata() + const pages = [] + + for ( + let pageNumber = 1; + pageNumber <= Math.min(maxPages, pdf.numPages); + pageNumber++ + ) { + pages.push(await pageLines(pdf, pageNumber)) + } + + const lines = pages.flat() + let images = [] + try { + images = await extractImages(pdf, 1) + } catch (_) {} + + return { + info: info || {}, + xmp: metadata?.getAll?.() || {}, + pageCount: pdf.numPages, + firstPageLines: pages[0] || [], + lines, + images, + text: lines.map(line => line.text).join('\n') + } + } finally { + try { + await pdf.loadingTask.destroy() + } catch (_) {} + } +} + +module.exports = { readDocument, toLines } diff --git a/packages/metascraper-pdf/src/embedded.js b/packages/metascraper-pdf/src/embedded.js new file mode 100644 index 000000000..a87d23393 --- /dev/null +++ b/packages/metascraper-pdf/src/embedded.js @@ -0,0 +1,153 @@ +'use strict' + +const JUNK_TITLE = [ + /^untitled/i, + /^arxiv:/i, + /^document ?\d*$/i, + /^microsoft word/i, + /^microsoft powerpoint/i, + /^print$/i, + /^slide ?\d*$/i, + /^layout ?\d*$/i, + /^book ?\d*$/i, + /\.(pdf|doc|docx|dot|ppt|pptx|xls|xlsx|tex|indd|qxd|pages|odt|rtf|md|htm|html|dvi)$/i, + /^[\da-f-]{16,}$/i, + /^[/\\~]/ +] + +const JUNK_AUTHOR = [ + /^(user|users|admin|administrator|owner|guest|unknown|anonymous|author|me|none|null|n\/?a)$/i, + /^(windows|microsoft|office|word|acrobat|adobe|wps|libreoffice|openoffice|hp|dell|toshiba|acer|lenovo|asus|sony|compaq)[\s-]*(user|office user|inc\.?)?$/i, + /\.(pdf|doc|docx|tex)$/i +] + +const SERIES_PREFIX = + /^[^:]{0,60}?\b(working paper|discussion paper|technical report|staff report)s?\s*\d*\s*:\s*/i +const CATALOG_PREFIX = /^(redalyc|scielo|dialnet)\.\s*/i +const VENUE_DESCRIPTION = + /^((19|20)\d{2}\s+)?(neural information|ieee|acm|proceedings|conference|workshop|journal|volume|doi|https?:|www\.)|https?:\/\/|\bdoi:\s*10\.\d/i + +const MIN_TITLE_LENGTH = 3 +const MIN_DESCRIPTION_LENGTH = 40 + +const clean = value => { + const text = Array.isArray(value) ? value.join(', ') : value + if (typeof text !== 'string') return null + const normalized = text.replace(/\s+/g, ' ').trim() + return normalized.length > 0 ? normalized : null +} + +const isJunk = (value, patterns) => + patterns.some(pattern => pattern.test(value)) + +const sameAsGenerator = (value, { creator, producer }) => + [creator, producer] + .filter(Boolean) + .some(generator => generator.toLowerCase() === value.toLowerCase()) + +const toDate = value => { + const match = + /^D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?(?:Z|z|([+-])(\d{2})'?(\d{2})?'?)?$/.exec( + String(value || '') + ) + if (!match) return null + const [ + , + year, + month = '01', + day = '01', + hour = '00', + minute = '00', + second = '00', + sign, + offsetHour = '00', + offsetMinute = '00' + ] = match + const y = Number(year) + const mo = Number(month) + const d = Number(day) + const h = Number(hour) + const mi = Number(minute) + const s = Number(second) + const oh = Number(offsetHour) + const om = Number(offsetMinute) + if ( + y < 1800 || + mo < 1 || + mo > 12 || + d < 1 || + d > 31 || + h > 23 || + mi > 59 || + s > 59 || + oh > 14 || + om > 59 + ) { + return null + } + let time = Date.UTC(y, mo - 1, d, h, mi, s) + const utc = new Date(time) + if ( + utc.getUTCFullYear() !== y || + utc.getUTCMonth() !== mo - 1 || + utc.getUTCDate() !== d + ) { + return null + } + if (sign === '+' || sign === '-') { + time -= (sign === '+' ? 1 : -1) * (oh * 60 + om) * 60 * 1000 + } + return new Date(time).toISOString() +} + +/** + * The Info dictionary and XMP packet, minus the noise every PDF writer leaves + * behind: LaTeX ships an empty title, Word ships the filename, conference + * templates ship the venue as the subject. + */ +const readEmbedded = ({ info = {}, xmp = {} } = {}) => { + const creator = clean(info.Creator) + const producer = clean(info.Producer) + const generators = { creator, producer } + + const rawTitle = (clean(xmp['dc:title']) || clean(info.Title) || '') + .replace(SERIES_PREFIX, '') + .replace(CATALOG_PREFIX, '') + const rawAuthor = clean(xmp['dc:creator']) || clean(info.Author) + const rawDescription = clean(xmp['dc:description']) || clean(info.Subject) + + const title = + rawTitle.length >= MIN_TITLE_LENGTH && + !isJunk(rawTitle, JUNK_TITLE) && + !sameAsGenerator(rawTitle, generators) + ? rawTitle + : null + + const author = + rawAuthor && + !isJunk(rawAuthor, JUNK_AUTHOR) && + !sameAsGenerator(rawAuthor, generators) + ? rawAuthor + : null + + const description = + rawDescription && + rawDescription.length >= MIN_DESCRIPTION_LENGTH && + !VENUE_DESCRIPTION.test(rawDescription) + ? rawDescription + : null + + return { + title, + author, + description, + publisher: clean(xmp['dc:publisher']), + date: toDate(info.CreationDate) || clean(xmp['xmp:createdate']), + lang: clean(xmp['dc:language']) || clean(info.Language) || clean(info.Lang), + creator, + producer, + keywords: clean(info.Keywords) + } +} + +module.exports = { readEmbedded, toDate } diff --git a/packages/metascraper-pdf/src/index.d.ts b/packages/metascraper-pdf/src/index.d.ts new file mode 100644 index 000000000..5f1b4e3c2 --- /dev/null +++ b/packages/metascraper-pdf/src/index.d.ts @@ -0,0 +1,30 @@ +declare function rules(options?: rules.Options): import('metascraper').Rules; + +declare namespace rules { + interface Options { + /** + * How many pages to read text from. Only the first page is used for the + * title, author and publisher; extra pages feed the description fallback. + * + * @default 2 + */ + maxPages?: number; + /** + * https://github.com/sindresorhus/got#options + */ + gotOpts?: import('got').Options; + /** + * https://github.com/microlinkhq/keyv/tree/master/packages/memoize#keyvoptions + */ + keyvOpts?: import('@keyvhq/core').Options; + /** + * Called to get the PDF bytes behind `url`. Defaults to a `got` download. + */ + getPdf?: (url: string) => ArrayBuffer | Buffer | Uint8Array | null | undefined | Promise; + } + + /** `true` when `url` points at a PDF document. */ + function test(props: { url?: string }): boolean; +} + +export = rules; diff --git a/packages/metascraper-pdf/src/index.js b/packages/metascraper-pdf/src/index.js new file mode 100644 index 000000000..8c2b9c253 --- /dev/null +++ b/packages/metascraper-pdf/src/index.js @@ -0,0 +1,196 @@ +'use strict' + +const asyncMemoizeOne = require('async-memoize-one') +const memoize = require('@keyvhq/memoize') +const got = require('got') + +const helpers = require('@metascraper/helpers') + +const { getAuthor, nameCount } = require('./author') +const { getDescription } = require('./description') +const { getDate } = require('./date') +const { getPublisher } = require('./publisher') +const { getTitle } = require('./title') +const { getLang } = require('./lang') +const { getMedia } = require('./media') +const { headerLines } = require('./layout') +const { readDocument } = require('./document') +const { readEmbedded } = require('./embedded') +const { comparable, isInvertedName } = require('./text') + +const PDF_MAGIC = Buffer.from('%PDF') +const PDF_HEAD = 1024 +const PDF_PATH = /(?:^|\/)pdf(?:\/|$)/i +const PDF_TYPE = /^(?:pdf|printable)$/i +const MAX_PAGES = 2 + +const toBytes = input => { + if (input instanceof ArrayBuffer) return new Uint8Array(input) + if (ArrayBuffer.isView(input)) { + return new Uint8Array(input.buffer, input.byteOffset, input.byteLength) + } + return null +} + +/** `%PDF` may sit anywhere in the first 1KB; the spec allows leading junk. */ +const isPdf = input => { + const bytes = toBytes(input) + return Boolean( + bytes && Buffer.from(bytes.subarray(0, PDF_HEAD)).includes(PDF_MAGIC) + ) +} + +const isPdfLink = url => { + if (!url || !helpers.isUrl(url)) return false + if (helpers.isPdfUrl(url)) return true + try { + const parsed = new URL(url) + return ( + PDF_PATH.test(parsed.pathname) || + PDF_TYPE.test(parsed.searchParams.get('type') || '') + ) + } catch (_) { + return false + } +} + +/** "Surname, Given" is one person; "A, B" is two. */ +const firstAuthor = value => { + if (!value) return null + const first = value.split(/\s*;\s*/)[0] + if (isInvertedName(first)) { + const [surname, given] = first.split(/,\s*/) + return `${given} ${surname}` + } + return first.split(/,\s*/)[0] +} + +const appearsIn = (value, text) => { + const needle = comparable(value) + return needle.length > 0 && comparable(text).includes(needle) +} + +/** A name lifted out of the title block is a title fragment, not an author. */ +const withoutContext = (names, context) => { + if (!names) return null + const kept = names + .split(', ') + .filter(name => !context.some(entry => entry && appearsIn(name, entry))) + return kept.length > 0 ? kept.join(', ') : null +} + +const extract = async ({ url, pdf, maxPages }) => { + const document = await readDocument(pdf, { maxPages }) + const embedded = readEmbedded(document) + const rawEmbedded = { + description: document.info.Subject, + keywords: document.info.Keywords, + title: document.info.Title + } + + const lines = headerLines(document.firstPageLines) + const layoutTitle = getTitle(lines) + + const title = + embedded.title && appearsIn(embedded.title, document.text) + ? embedded.title + : layoutTitle?.text || embedded.title + + const layoutAuthor = withoutContext( + getAuthor(lines, { + titleIndexes: layoutTitle?.indexes || [] + }), + [title] + ) + const authors = + nameCount(layoutAuthor) > nameCount(embedded.author) + ? layoutAuthor + : embedded.author || layoutAuthor + + const author = firstAuthor(authors) + const publisher = + embedded.publisher || getPublisher(lines, { url, title, author: authors }) + const description = embedded.description || getDescription(document.lines) + const date = getDate(url, { + lines: document.firstPageLines, + text: document.text, + embedded, + rawEmbedded + }) + const lang = getLang(document.text, { url, embedded }) + const { image, logo } = getMedia(document.images, { url }) + + return { + title, + author, + authors, + description, + publisher, + date, + lang, + image, + logo + } +} + +const defaultGetPdf = gotOpts => async url => { + try { + const { body } = await got(url, { responseType: 'buffer', ...gotOpts }) + return isPdf(body) ? body : null + } catch (_) { + return null + } +} + +const createLoad = ({ maxPages, gotOpts, keyvOpts, getPdf }) => { + const fetchPdf = getPdf || defaultGetPdf(gotOpts) + + const parse = async url => { + const pdf = toBytes(await fetchPdf(url)) + if (!isPdf(pdf)) return {} + return extract({ url, pdf, maxPages }) + } + + return asyncMemoizeOne( + memoize(parse, keyvOpts, { + value: value => (value === undefined ? null : value) + }) + ) +} + +const NORMALIZERS = { + author: helpers.author, + date: helpers.date, + description: helpers.description, + image: helpers.image, + lang: helpers.lang, + logo: helpers.logo, + publisher: helpers.publisher, + title: helpers.title +} + +const fromPdf = + (propName, load) => + async ({ url }) => { + const metadata = await load(url) + return NORMALIZERS[propName](metadata?.[propName], { url }) + } + +module.exports = ({ maxPages = MAX_PAGES, gotOpts, keyvOpts, getPdf } = {}) => { + const load = createLoad({ maxPages, gotOpts, keyvOpts, getPdf }) + + return { + pkgName: 'metascraper-pdf', + test: ({ url }) => isPdfLink(url), + author: [fromPdf('author', load)], + date: [fromPdf('date', load)], + description: [fromPdf('description', load)], + image: [fromPdf('image', load)], + lang: [fromPdf('lang', load)], + logo: [fromPdf('logo', load)], + publisher: [fromPdf('publisher', load)], + title: [fromPdf('title', load)] + } +} + +module.exports.test = ({ url }) => isPdfLink(url) diff --git a/packages/metascraper-pdf/src/lang.js b/packages/metascraper-pdf/src/lang.js new file mode 100644 index 000000000..49e91734a --- /dev/null +++ b/packages/metascraper-pdf/src/lang.js @@ -0,0 +1,35 @@ +'use strict' + +const { lang, parseUrl } = require('@metascraper/helpers') + +const HOST_LANG = { + redalyc: 'es', + scielo: 'es' +} + +const EN = + '\\b(the|and|of|to|in|for|with|this|that|from|are|was|we|is|on|as|by|an|a|be|or|it)\\b' +const ES = + '\\b(el|la|los|las|del|una|para|con|por|que|este|esta|como|más|un|se|al)\\b' + +const hostLang = url => { + const { domainWithoutSuffix } = parseUrl(url) || {} + return HOST_LANG[domainWithoutSuffix] || null +} + +const count = (text, pattern) => + (text.match(new RegExp(pattern, 'gi')) || []).length + +const fromText = text => { + const sample = String(text || '').slice(0, 2000) + const en = count(sample, EN) + const es = count(sample, ES) + if (es > en && es >= 8) return 'es' + if (en >= 4) return 'en' + return null +} + +const getLang = (text, { url, embedded } = {}) => + lang(embedded?.lang) || hostLang(url) || fromText(text) + +module.exports = { getLang } diff --git a/packages/metascraper-pdf/src/layout.js b/packages/metascraper-pdf/src/layout.js new file mode 100644 index 000000000..abbfabc38 --- /dev/null +++ b/packages/metascraper-pdf/src/layout.js @@ -0,0 +1,68 @@ +'use strict' + +const MAX_HEADER_LINES = 20 +const MAX_FOOTER_LINES = 8 +const MAX_FOOTER_WORDS = 30 +const MAX_PROMINENT_LINES = 8 +const PROMINENT_SIZE_LEVELS = 2 +const FOLLOWING_LINES = 2 +const FOOTER_BAND = 0.12 +const BODY_LINE_WORDS = 18 + +const wordCount = text => text.split(/\s+/).length + +const leadingBlock = lines => { + const header = [] + + for (const line of lines) { + if (header.length >= MAX_HEADER_LINES) break + header.push(line) + if (wordCount(line.text) >= BODY_LINE_WORDS && header.length > 1) break + } + + return header +} + +const prominentLines = lines => { + const sizes = [...new Set(lines.map(line => line.size))] + .sort((left, right) => right - left) + .slice(0, PROMINENT_SIZE_LEVELS) + + return lines + .filter(line => sizes.includes(line.size)) + .slice(0, MAX_PROMINENT_LINES) + .flatMap(line => + lines.slice(line.pageIndex, line.pageIndex + 1 + FOLLOWING_LINES) + ) +} + +/** + * The lines worth reading on the first page: the block above the first + * paragraph, the largest type anywhere on the page (magazine layouts put the + * title well below the fold of the text stream), and the running footer. + */ +const headerLines = lines => { + const numbered = lines.map((line, pageIndex) => ({ ...line, pageIndex })) + const leading = leadingBlock(numbered) + const pageHeight = numbered.length > 0 ? numbered[0].pageHeight : null + + const isFooter = line => + pageHeight + ? line.y <= pageHeight * FOOTER_BAND + : line.pageIndex >= numbered.length - MAX_FOOTER_LINES + + const footer = numbered + .filter(line => isFooter(line) && wordCount(line.text) < MAX_FOOTER_WORDS) + .slice(-MAX_FOOTER_LINES) + + const chosen = new Map() + for (const line of [...leading, ...prominentLines(numbered), ...footer]) { + chosen.set(line.pageIndex, line) + } + + return [...chosen.values()] + .sort((left, right) => left.pageIndex - right.pageIndex) + .map((line, index) => ({ ...line, index })) +} + +module.exports = { headerLines } diff --git a/packages/metascraper-pdf/src/media.js b/packages/metascraper-pdf/src/media.js new file mode 100644 index 000000000..9e9c16cee --- /dev/null +++ b/packages/metascraper-pdf/src/media.js @@ -0,0 +1,92 @@ +'use strict' + +const { deflateSync } = require('node:zlib') + +const MIN_SIDE = 16 +const MAX_PIXELS = 400 * 400 +const MAX_LOGO_SIDE = 256 + +const crcTable = new Uint32Array(256) +for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + crcTable[n] = c >>> 0 +} + +const crc32 = buf => { + let crc = 0xffffffff + for (let i = 0; i < buf.length; i++) { + crc = crcTable[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8) + } + return (crc ^ 0xffffffff) >>> 0 +} + +const chunk = (type, data) => { + const length = Buffer.alloc(4) + length.writeUInt32BE(data.length) + const body = Buffer.concat([Buffer.from(type), data]) + const crc = Buffer.alloc(4) + crc.writeUInt32BE(crc32(body)) + return Buffer.concat([length, body, crc]) +} + +/** Encode raw RGB/RGBA from unpdf into a PNG data URI. node:zlib only. */ +const toPngDataUri = ({ width, height, channels, data }) => { + const source = Buffer.from(data.buffer, data.byteOffset, data.byteLength) + const stride = width * channels + const raw = Buffer.alloc((stride + 1) * height) + for (let y = 0; y < height; y++) { + source.copy(raw, y * (stride + 1) + 1, y * stride, y * stride + stride) + } + const ihdr = Buffer.alloc(13) + ihdr.writeUInt32BE(width, 0) + ihdr.writeUInt32BE(height, 4) + ihdr[8] = 8 + ihdr[9] = channels === 4 ? 6 : 2 + const png = Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + chunk('IHDR', ihdr), + chunk('IDAT', deflateSync(raw)), + chunk('IEND', Buffer.alloc(0)) + ]) + return `data:image/png;base64,${png.toString('base64')}` +} + +const usable = images => + images.filter( + img => + img && + img.data && + img.width >= MIN_SIDE && + img.height >= MIN_SIDE && + img.width * img.height <= MAX_PIXELS && + (img.channels === 3 || img.channels === 4) + ) + +const isLogo = img => + img.width <= MAX_LOGO_SIDE && + img.height <= MAX_LOGO_SIDE && + img.width / img.height >= 0.4 && + img.width / img.height <= 2.5 + +const favicon = url => + `https://www.google.com/s2/favicons?domain_url=${encodeURIComponent( + url + )}&sz=128` + +const getMedia = (images, { url } = {}) => { + const candidates = usable(images).sort( + (left, right) => right.width * right.height - left.width * left.height + ) + + const logoImage = candidates.findLast(isLogo) + const picture = candidates.find(img => img !== logoImage) || logoImage + const logo = logoImage ? toPngDataUri(logoImage) : url ? favicon(url) : null + + return { + image: picture ? toPngDataUri(picture) : null, + logo + } +} + +module.exports = { favicon, getMedia, toPngDataUri, usable } diff --git a/packages/metascraper-pdf/src/publisher.js b/packages/metascraper-pdf/src/publisher.js new file mode 100644 index 000000000..f4822ae1c --- /dev/null +++ b/packages/metascraper-pdf/src/publisher.js @@ -0,0 +1,120 @@ +'use strict' + +const { parseUrl } = require('@metascraper/helpers') + +const { comparable, flatten, isBannerLine, tidy } = require('./text') + +const PUBLISHER_NOISE = + /https?:\/\/\S+|\bwww\.\S+|\bdoi:\S+|\b10\.\d{4,9}\/\S+|\(\s*\d{4}\s*\)|,?\s*\bpages?\b.*$|\b\d[\d:;,.()-]*\b/gi +const PUBLISHER_SEGMENTS = /\s*[|·•‖]\s*/ +const LETTER_SPACED = /\b(?:\p{L}\s){3,}\p{L}\b/gu +const RIGHTS_NOTICE = /©|\(c\)|\ball rights reserved\b\.?/gi +const WORKFLOW_PREFIX = + /^(published|received|accepted|submitted|revised|edited|reviewed|updated|available)\b/i +const STRUCTURAL_WORD = + /^(january|february|march|april|may|june|july|august|september|october|november|december|winter|spring|summer|fall|autumn|volume|vol|issue|no|number|article|page|pp|supplement|edition|part|series)$/i + +const VENUE_OPENING = + /^(proceedings|journal|transactions|communications|frontiers|revista|revue|zeitschrift|annals|acta|bulletin|nature|scientific reports|plos|ieee|acm|arxiv|biorxiv|the journal)\b/i +const VENUE_MARK = /\b(publish\w+|proceedings|conference|symposium)\b/i +const AFFILIATION = + /\b(universi\w*|department|departamento|dept|laborator\w*|labs?|faculty|school|college|institut\w*|hospital|clinic|cent(er|re|ro)|research|group|team|division)\b/i + +const CITY_STATE = /^\p{Lu}[\p{L}.' -]+,\s*[A-Z]{2}$/u + +const HOST_PUBLISHER = { + arxiv: 'arXiv', + nber: 'NBER', + jmlr: 'JMLR', + thecvf: 'CVF', + neurips: 'NeurIPS', + plos: 'PLOS', + nature: 'Nature', + elifesciences: 'eLife', + frontiersin: 'Frontiers', + aclanthology: 'ACL Anthology', + redalyc: 'Redalyc', + 'ceur-ws': 'CEUR-WS', + springer: 'Springer', + biomedcentral: 'BMC', + bis: 'BIS', + bitcoin: 'Bitcoin', + berkshirehathaway: 'Berkshire Hathaway', + openai: 'OpenAI' +} + +const MAX_PUBLISHER_WORDS = 12 + +const collapseLetterSpacing = text => + text.replace(LETTER_SPACED, match => match.replace(/\s+/g, '')) + +const hostName = url => { + const { domainWithoutSuffix, hostname } = parseUrl(url) || {} + return domainWithoutSuffix || hostname || null +} + +const publisherFromUrl = url => { + const { domainWithoutSuffix, hostname } = parseUrl(url) || {} + if (HOST_PUBLISHER[domainWithoutSuffix]) { + return HOST_PUBLISHER[domainWithoutSuffix] + } + const name = domainWithoutSuffix || hostname || url + return name.length > 1 ? name[0].toUpperCase() + name.slice(1) : name +} + +const matchesHost = (value, url) => { + const host = hostName(url) + if (!host) return false + return ( + comparable(value).startsWith(comparable(host)) || + comparable(host).startsWith(comparable(value)) + ) +} + +const isVenue = (value, url) => + Boolean(value) && + (VENUE_OPENING.test(value) || + VENUE_MARK.test(value) || + matchesHost(value, url)) + +/** + * Running headers and footers carry the venue next to volume, doi and page + * numbers: `Scientific Reports | (2023) 13:1234 | https://doi.org/...`. + */ +const publisherFromLine = text => + flatten(collapseLetterSpacing(text).replace(RIGHTS_NOTICE, ' ')) + .split(PUBLISHER_SEGMENTS) + .map(part => flatten(tidy(part.replace(PUBLISHER_NOISE, ' ')))) + .filter(part => { + const words = part.split(/\s+/).filter(Boolean) + if (words.length === 0 || words.length > MAX_PUBLISHER_WORDS) return false + if (WORKFLOW_PREFIX.test(part)) return false + return ( + /\p{L}{3}/u.test(part) && + !words.every(word => STRUCTURAL_WORD.test(word)) + ) + })[0] || null + +const sameText = (left, right) => + Boolean(left && right) && comparable(left) === comparable(right) + +const isUsablePublisher = (value, { author, title }) => + Boolean(value) && + !sameText(value, author) && + !sameText(value, title) && + !AFFILIATION.test(value) && + !CITY_STATE.test(value) + +const getPublisher = (lines, { url, title, author }) => { + const venue = lines + .filter(line => !isBannerLine(line.text)) + .map(line => publisherFromLine(line.text)) + .find( + value => + isVenue(value, url) && isUsablePublisher(value, { author, title }) + ) + + return venue || publisherFromUrl(url) +} + +module.exports = { getPublisher, isVenue, publisherFromLine, publisherFromUrl } diff --git a/packages/metascraper-pdf/src/text.js b/packages/metascraper-pdf/src/text.js new file mode 100644 index 000000000..d216a6a4c --- /dev/null +++ b/packages/metascraper-pdf/src/text.js @@ -0,0 +1,152 @@ +'use strict' + +const EMAIL = /\S+@\S+/ +const EMAIL_GLOBAL = /\S+@\S+/g +const FOOTNOTE_MARKS = /[*∗†‡§¶#]/g +const SUPERSCRIPTS = /(?<=\p{L})\d+(?:,\d+)*(?=\W|$)/gu +const COUNTRY_ABBREVIATION = /(^|\s)\p{Lu}\.\p{Lu}\.?($|\s|,)/u +const PLACE_NAME = + /^(costa rica|united states|united kingdom|new zealand|south africa|puerto rico|el salvador|saudi arabia|south korea|hong kong)$/i +const INVERTED_NAME = + /^\p{Lu}[\p{L}'’-]+(?:\s\p{Lu}[\p{L}'’-]+)*,\s\p{Lu}[\p{L}'’-]+(?:\s\p{Lu}[\p{L}'’.-]*)*$/u +const NAME_RUN_BOUNDARY = /(?<=\p{Ll})\s(?=\p{Lu}[\p{Ll}])/u + +const ORGANIZATION_WORDS = + /\b(universi\w*|institut\w*|department\w*|departamento|dept|laborator\w*|labs?|college|school|escuela|academy|academia|research|brain|cent(er|re|ro)|foundation|fundaci\w*|hospital|ministry|ministerio|agency|agencia|association|society|press|journal|proceedings|conference|inc|llc|ltd|gmbh|corp|corporation|company|co|google|microsoft|facebook|meta|openai|deepmind|nvidia|amazon|apple|ibm|baidu|alibaba|tencent|huawei|samsung|intel|adobe|anthropic)\b/i + +/** Lower case only: a middle initial is `A`, a function word is `and`. */ +const FUNCTION_WORDS = + /\b(for|and|the|with|in|on|to|at|of|from|via|using|towards?|a|an|is|are|be|by)\b/ + +const NAME_PARTICLES = new Set([ + 'van', + 'von', + 'de', + 'del', + 'della', + 'di', + 'da', + 'dos', + 'der', + 'den', + 'la', + 'le', + 'bin', + 'ibn' +]) + +const BANNER_LINE = + /^((nber|bis|ecb|imf|oecd)\s+)?(working|discussion|conference|staff)\s+papers?(\s+series)?\s*\d*\b|^technical report$|^(review|research|original research|short|regular)\s+(article|paper)s?$|^preprints?$|^no\.?\s*\d+$|^volume\s+\d+|^published as a conference paper|^arxiv:\S+|^submitted to |^(reviews?|articles?|letters?|analysis|perspectives?|comments?|editorials?|news|features?|insights?|reports?|columns?|correspondence)$/i + +const MAX_NAME_WORDS = 6 +const MAX_INVERTED_WORDS = 3 +const MIN_GIVEN_LETTERS = 3 +const MIN_RUN_WORDS = 4 +const MAX_RUN_WORDS = 14 + +const tidy = value => + value + .replace(/[^\S ]+/g, ' ') + .replace(/ {3,}/g, ' ') + .trim() + .replace(/^[,;:.\-–—&]+\s*|[,;:.\-–—&]+$/g, '') + +const flatten = value => value.replace(/\s+/g, ' ').trim() + +const comparable = value => + String(value || '') + .toLowerCase() + .replace(/[^a-z0-9]/g, '') + +const stripNoise = text => + tidy( + text + .replace(EMAIL_GLOBAL, ' ') + .replace(FOOTNOTE_MARKS, ' ') + .replace(SUPERSCRIPTS, '') + ) + +const isShouting = text => text === text.toUpperCase() && /\p{Lu}/u.test(text) + +const isCapitalized = word => + /^[\p{Lu}]/u.test(word) || NAME_PARTICLES.has(word.toLowerCase()) + +const isInvertedName = text => { + if ( + !INVERTED_NAME.test(text) || + text.split(/\s+/).length > MAX_INVERTED_WORDS + ) { + return false + } + const given = text.split(',')[1] || '' + return (given.match(/\p{L}/gu) || []).length >= MIN_GIVEN_LETTERS +} + +const isPersonName = rawText => { + const text = stripNoise(rawText) + if (isInvertedName(text)) return !ORGANIZATION_WORDS.test(text) + if (EMAIL.test(text) || /\d/.test(text) || /^https?:/i.test(text)) { + return false + } + if ( + isShouting(text) || + COUNTRY_ABBREVIATION.test(text) || + PLACE_NAME.test(text) + ) { + return false + } + if (ORGANIZATION_WORDS.test(text) || FUNCTION_WORDS.test(text)) return false + const words = text.split(/\s+/) + return ( + words.length >= 2 && + words.length <= MAX_NAME_WORDS && + words.every(isCapitalized) + ) +} + +const isBannerLine = text => BANNER_LINE.test(tidy(text)) + +const splitNamePairs = text => { + const words = text.split(/\s+/).filter(Boolean) + if ( + words.length < MIN_RUN_WORDS || + words.length > MAX_RUN_WORDS || + words.length % 2 !== 0 + ) { + return [] + } + if (!words.every(word => /^\p{Lu}/u.test(word))) return [] + return words.flatMap((word, index) => + index % 2 === 0 ? [`${word} ${words[index + 1]}`] : [] + ) +} + +const splitNameRun = part => { + if (part.split(/\s+/).length < MIN_RUN_WORDS) return [part] + const pieces = part.split(NAME_RUN_BOUNDARY) + return pieces.every( + piece => /^\p{Lu}/u.test(piece) && piece.split(/\s+/).length <= 3 + ) + ? pieces + : [part] +} + +const splitNames = text => + isInvertedName(text) + ? [text] + : text.split(/\s{2,}|\s*(?:,|;| and | & )\s*/).flatMap(splitNameRun) + +module.exports = { + EMAIL, + ORGANIZATION_WORDS, + PLACE_NAME, + comparable, + flatten, + isBannerLine, + isInvertedName, + isPersonName, + splitNamePairs, + splitNames, + stripNoise, + tidy +} diff --git a/packages/metascraper-pdf/src/title.js b/packages/metascraper-pdf/src/title.js new file mode 100644 index 000000000..27f59c563 --- /dev/null +++ b/packages/metascraper-pdf/src/title.js @@ -0,0 +1,89 @@ +'use strict' + +const { EMAIL, flatten, isBannerLine, isPersonName } = require('./text') + +const TITLE_BLOCK_LIMIT = 6 +const TITLE_SIZE_LEVELS = 3 +const BYLINE_DISTANCE = 4 +const MIN_TITLE_LENGTH = 3 +const MAX_TITLE_LENGTH = 300 + +const CONTINUES_TITLE = + /\b(for|of|and|the|in|on|to|with|from|a|an|at|by|via|using|towards?|de|del|la)$/i +const LINE_NUMBER = /(?<=\p{L}{3,})\d{1,2}(?=\s|$)/gu + +const distinctSizes = lines => + [...new Set(lines.map(line => line.size))].sort((left, right) => right - left) + +/** + * A title keeps its font size while it wraps, and ends where the byline starts. + */ +const titleLines = (lines, titleLine) => { + const block = [titleLine] + + for (let offset = 1; offset < TITLE_BLOCK_LIMIT; offset++) { + const next = lines[titleLine.index + offset] + const previous = block[block.length - 1] + if (!next || next.size !== titleLine.size) break + if (next.pageIndex != null && next.pageIndex !== previous.pageIndex + 1) { + break + } + if (/[.?!]$/.test(previous.text) && !/:$/.test(previous.text)) break + const continues = CONTINUES_TITLE.test(previous.text) + if (EMAIL.test(next.text)) break + if (!continues && (isPersonName(next.text) || isBannerLine(next.text))) { + break + } + block.push(next) + } + + return block +} + +/** A line of names, rather than a title, has more names set beside it. */ +const isByline = (line, lines) => + isPersonName(line.text) && + lines.some( + other => + other.index !== line.index && + other.size === line.size && + Math.abs(other.index - line.index) <= BYLINE_DISTANCE && + isPersonName(other.text) + ) + +/** + * The title is the largest type on the page, skipping the banners publishers + * print above it (`NBER WORKING PAPER SERIES`, `arXiv:2303.08774v6`, `REVIEW`) + * and any byline set in the same size. + */ +const findTitleLine = lines => { + for (const size of distinctSizes(lines).slice(0, TITLE_SIZE_LEVELS)) { + const line = lines.find( + candidate => + candidate.size === size && + !isBannerLine(candidate.text) && + !isByline(candidate, lines) + ) + if (line) return line + } + return null +} + +const isUsableTitle = text => + typeof text === 'string' && + text.length >= MIN_TITLE_LENGTH && + text.length <= MAX_TITLE_LENGTH + +const getTitle = lines => { + const line = findTitleLine(lines) + if (!line) return null + const block = titleLines(lines, line) + const text = flatten( + block.map(entry => entry.text.replace(LINE_NUMBER, '')).join(' ') + ) + return isUsableTitle(text) + ? { text, indexes: block.map(entry => entry.index) } + : null +} + +module.exports = { getTitle, isByline } diff --git a/packages/metascraper-pdf/test/fixtures.js b/packages/metascraper-pdf/test/fixtures.js new file mode 100644 index 000000000..c0deacc2f --- /dev/null +++ b/packages/metascraper-pdf/test/fixtures.js @@ -0,0 +1,42 @@ +'use strict' + +const { existsSync, readFileSync } = require('fs') +const { readFile } = require('fs/promises') +const { resolve } = require('path') +const test = require('ava').default + +const FIXTURES = resolve(__dirname, 'fixtures') +const skipReason = existsSync(resolve(FIXTURES, '1.pdf')) + ? null + : 'PDF fixtures missing; run test/fixtures/download.sh' + +const urls = existsSync(resolve(FIXTURES, 'urls.txt')) + ? readFileSync(resolve(FIXTURES, 'urls.txt'), 'utf8').trim().split('\n') + : [] + +const getPdf = async url => { + const index = urls.indexOf(url) + return readFile(resolve(FIXTURES, `${index + 1}.pdf`)) +} + +const metascraper = require('metascraper')([require('..')({ getPdf })]) + +const summarize = metadata => { + const compact = value => + typeof value === 'string' && value.startsWith('data:') + ? `${value.slice(0, 21)}…${value.length}` + : value + return { + ...metadata, + image: compact(metadata.image), + logo: compact(metadata.logo) + } +} + +const run = skipReason ? test.skip : test + +for (const url of urls) { + run(url, async t => { + t.snapshot(summarize(await metascraper({ url }))) + }) +} diff --git a/packages/metascraper-pdf/test/fixtures/.gitignore b/packages/metascraper-pdf/test/fixtures/.gitignore new file mode 100644 index 000000000..a13633799 --- /dev/null +++ b/packages/metascraper-pdf/test/fixtures/.gitignore @@ -0,0 +1 @@ +*.pdf diff --git a/packages/metascraper-pdf/test/fixtures/download.sh b/packages/metascraper-pdf/test/fixtures/download.sh new file mode 100755 index 000000000..ad2e4d85a --- /dev/null +++ b/packages/metascraper-pdf/test/fixtures/download.sh @@ -0,0 +1,2 @@ +cd "$(dirname "$0")" || exit 1 +nl -ba urls.txt | xargs -n 2 -P 8 sh -c 'curl -fL --retry 3 --retry-all-errors -o "$0.pdf" "$1"' diff --git a/packages/metascraper-pdf/test/fixtures/urls.txt b/packages/metascraper-pdf/test/fixtures/urls.txt new file mode 100644 index 000000000..cd72a0978 --- /dev/null +++ b/packages/metascraper-pdf/test/fixtures/urls.txt @@ -0,0 +1,53 @@ +https://www.nber.org/system/files/working_papers/w28110/w28110.pdf +https://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf +https://proceedings.neurips.cc/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf +https://www.stat.berkeley.edu/~breiman/randomforest2001.pdf +https://homes.cs.washington.edu/~pedrod/papers/cacm12.pdf +https://arxiv.org/pdf/1706.03762v7 +https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0230416&type=printable +https://aclanthology.org/N19-1423.pdf +https://jmlr.org/papers/volume15/srivastava14a/srivastava14a.pdf +https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0000308&type=printable +https://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf +https://www.frontiersin.org/articles/10.3389/fpsyg.2019.00001/pdf +https://www.nature.com/articles/s41598-023-28020-5.pdf +https://link.springer.com/content/pdf/10.1186/s13059-020-02007-1.pdf +https://www.nber.org/system/files/working_papers/w31161/w31161.pdf +https://cdn.elifesciences.org/articles/00013/elife-00013-v1.pdf +https://aclanthology.org/2020.acl-main.703.pdf +https://arxiv.org/pdf/2303.08774v6 +https://www.redalyc.org/pdf/440/44029444004.pdf +https://ceur-ws.org/Vol-2600/paper1.pdf +https://arxiv.org/pdf/1512.03385v1 +https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf +https://journals.plos.org/plosbiology/article/file?id=10.1371/journal.pbio.3000410&type=printable +https://arxiv.org/pdf/2005.14165v4 +https://arxiv.org/pdf/1412.6980v9 +https://arxiv.org/pdf/2010.11929v2 +https://arxiv.org/pdf/1810.04805v2 +https://arxiv.org/pdf/2203.02155v1 +https://arxiv.org/pdf/cs/0102004v1 +https://arxiv.org/pdf/2104.08663v4 +https://www.jmlr.org/papers/volume12/pedregosa11a/pedregosa11a.pdf +https://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf +https://openaccess.thecvf.com/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf +https://openaccess.thecvf.com/content_ICCV_2017/papers/He_Mask_R-CNN_ICCV_2017_paper.pdf +https://aclanthology.org/D19-1410.pdf +https://aclanthology.org/P16-1162.pdf +https://proceedings.neurips.cc/paper/2015/file/14bfa6bb14875e45bba028a21ed38046-Paper.pdf +https://www.nber.org/system/files/working_papers/w26947/w26947.pdf +https://www.nber.org/system/files/working_papers/w30957/w30957.pdf +https://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.1003583&type=printable +https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1005510&type=printable +https://cdn.elifesciences.org/articles/57443/elife-57443-v2.pdf +https://www.frontiersin.org/articles/10.3389/fnins.2020.00001/pdf +https://www.nature.com/articles/s41598-020-69250-1.pdf +https://www.nature.com/articles/s41467-020-17419-7.pdf +https://link.springer.com/content/pdf/10.1186/s12874-020-01057-0.pdf +https://ceur-ws.org/Vol-3226/paper1.pdf +https://www.redalyc.org/pdf/567/56712871011.pdf +https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/s12859-020-3418-9.pdf +https://www.bis.org/publ/work1000.pdf +https://bitcoin.org/bitcoin.pdf +https://www.berkshirehathaway.com/letters/2023ltr.pdf +https://cdn.openai.com/papers/gpt-4.pdf diff --git a/packages/metascraper-pdf/test/helpers/index.js b/packages/metascraper-pdf/test/helpers/index.js new file mode 100644 index 000000000..7eebc223e --- /dev/null +++ b/packages/metascraper-pdf/test/helpers/index.js @@ -0,0 +1,86 @@ +'use strict' + +const PAGE_WIDTH = 612 +const PAGE_HEIGHT = 792 +const TOP_MARGIN = 72 +const LEFT_MARGIN = 72 + +const escapeText = text => text.replace(/([\\()])/g, '\\$1') + +const toStream = lines => { + let cursor = PAGE_HEIGHT - TOP_MARGIN + const operations = [] + + for (const { text, size = 10, gap = 6 } of lines) { + cursor -= size + gap + operations.push( + `BT /F1 ${size} Tf ${LEFT_MARGIN} ${cursor.toFixed(2)} Td (${escapeText( + text + )}) Tj ET` + ) + } + + return operations.join('\n') +} + +/** + * Builds a one page PDF out of `{ text, size }` lines, so tests exercise the + * real parser without committing binaries to the repository. + */ +const createPdf = (lines, { info = {} } = {}) => { + const stream = toStream(lines) + const infoEntries = Object.entries(info) + .map(([key, value]) => `/${key} (${escapeText(String(value))})`) + .join(' ') + + const objects = [ + '<< /Type /Catalog /Pages 2 0 R >>', + '<< /Type /Pages /Kids [3 0 R] /Count 1 >>', + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 ${PAGE_WIDTH} ${PAGE_HEIGHT}] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>`, + `<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`, + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>', + `<< ${infoEntries} >>` + ] + + let pdf = '%PDF-1.4\n' + const offsets = [] + + objects.forEach((body, index) => { + offsets.push(pdf.length) + pdf += `${index + 1} 0 obj\n${body}\nendobj\n` + }) + + const startxref = pdf.length + pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n` + for (const offset of offsets) { + pdf += `${String(offset).padStart(10, '0')} 00000 n \n` + } + pdf += `trailer\n<< /Size ${ + objects.length + 1 + } /Root 1 0 R /Info 6 0 R >>\nstartxref\n${startxref}\n%%EOF\n` + + return Buffer.from(pdf, 'latin1') +} + +const { default: listen } = require('async-listen') +const { createServer } = require('http') + +const closeServer = server => + require('util').promisify(server.close.bind(server))() + +const runServer = async (t, handler) => { + const server = createServer(async (req, res) => { + try { + await handler({ req, res }) + } catch (error) { + console.error(error) + res.statusCode = 500 + res.end() + } + }) + const url = await listen(server, { port: 0, host: '127.0.0.1' }) + t.teardown(() => closeServer(server)) + return url.toString() +} + +module.exports = { createPdf, runServer } diff --git a/packages/metascraper-pdf/test/index.js b/packages/metascraper-pdf/test/index.js new file mode 100644 index 000000000..d4c9c3863 --- /dev/null +++ b/packages/metascraper-pdf/test/index.js @@ -0,0 +1,166 @@ +'use strict' + +const test = require('ava').default + +const { createPdf, runServer } = require('./helpers') + +const { test: isPdfLink } = require('..') + +const createMetascraper = (pdf, opts) => + require('metascraper')([require('..')({ getPdf: async () => pdf, ...opts })]) + +const PAPER = createPdf( + [ + { text: 'Attention Is All You Need', size: 18 }, + { text: 'Ashish Vaswani Noam Shazeer Niki Parmar', size: 11 }, + { text: 'Google Brain', size: 11 }, + { text: 'avaswani@google.com', size: 9 }, + { text: 'Abstract', size: 11 }, + { + text: 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks.', + size: 9 + }, + { + text: 'We propose the Transformer, based solely on attention mechanisms.', + size: 9 + } + ], + { info: { Title: '', Author: '', Creator: 'LaTeX with hyperref' } } +) + +const WORKING_PAPER = createPdf( + [ + { text: 'NBER WORKING PAPER SERIES', size: 12 }, + { text: 'GENERATIVE AI AT WORK', size: 12 }, + { text: 'Erik Brynjolfsson', size: 12 }, + { text: 'Danielle Li', size: 12 }, + { text: 'Working Paper 31161', size: 12 }, + { text: 'Abstract', size: 10 }, + { + text: 'We study the staggered introduction of a generative AI assistant among customer support agents.', + size: 10 + } + ], + { info: { Title: 'printmgr file' } } +) + +const JOURNAL = createPdf([ + { text: 'Frontiers in Psychology | Volume 10 | Article 1', size: 7 }, + { + text: 'Institutional Violence Against Users of the Family Law Courts', + size: 16 + }, + { text: 'Miguel Clemente, Dolores Padilla-Racero', size: 10 }, + { text: 'Abstract', size: 10 }, + { + text: 'This work analyses the psychological consequences of institutional violence in family law courts.', + size: 9 + } +]) + +test('test() only accepts a PDF url', t => { + t.true(isPdfLink({ url: 'https://arxiv.org/pdf/1706.03762v7' })) + t.true(isPdfLink({ url: 'https://example.com/paper.pdf' })) + t.true( + isPdfLink({ + url: 'https://journals.plos.org/plosone/article/file?id=10.1371/x&type=printable' + }) + ) + t.true( + isPdfLink({ + url: 'https://www.frontiersin.org/articles/10.3389/fpsyg.2019.00001/pdf' + }) + ) + t.false(isPdfLink({ url: 'https://arxiv.org/abs/1706.03762' })) + t.false(isPdfLink({ url: 'https://example.com' })) + t.false(isPdfLink({})) +}) + +test('reads the metadata of a paper', async t => { + const metascraper = createMetascraper(PAPER) + const metadata = await metascraper({ + url: 'https://arxiv.org/pdf/1706.03762v7' + }) + + t.is(metadata.title, 'Attention Is All You Need') + t.is(metadata.author, 'Ashish Vaswani') + t.is(metadata.publisher, 'arXiv') + t.is(metadata.date, '2017-06-01T00:00:00.000Z') + t.is(metadata.lang, 'en') + t.is( + metadata.logo, + `https://www.google.com/s2/favicons?domain_url=${encodeURIComponent( + 'https://arxiv.org/pdf/1706.03762v7' + )}&sz=128` + ) + t.true( + metadata.description.startsWith('The dominant sequence transduction models') + ) +}) + +test('reads a PDF with leading junk or an ArrayBuffer', async t => { + const url = 'https://arxiv.org/pdf/1706.03762v7' + const junk = Buffer.concat([Buffer.from('\0\0'), PAPER]) + const fromJunk = await createMetascraper(junk)({ url }) + t.is(fromJunk.title, 'Attention Is All You Need') + + const copy = Uint8Array.from(PAPER) + const fromArrayBuffer = await createMetascraper(copy.buffer)({ url }) + t.is(fromArrayBuffer.title, 'Attention Is All You Need') +}) + +test('skips the banner a working paper prints above its title', async t => { + const metascraper = createMetascraper(WORKING_PAPER) + const metadata = await metascraper({ + url: 'https://www.nber.org/system/files/working_papers/w31161/w31161.pdf' + }) + + t.is(metadata.title, 'GENERATIVE AI AT WORK') + t.is(metadata.author, 'Erik Brynjolfsson') + t.is(metadata.publisher, 'NBER') + t.is(metadata.lang, 'en') +}) + +test('reads the journal out of the running header', async t => { + const metascraper = createMetascraper(JOURNAL) + const metadata = await metascraper({ + url: 'https://www.frontiersin.org/articles/10.3389/fpsyg.2019.00001/pdf' + }) + + t.is(metadata.publisher, 'Frontiers in Psychology') + t.is(metadata.author, 'Miguel Clemente') + t.is( + metadata.title, + 'Institutional Violence Against Users of the Family Law Courts' + ) +}) + +test('fetches the PDF from the url', async t => { + const origin = await runServer(t, ({ res }) => { + res.setHeader('content-type', 'application/pdf') + res.end(PAPER) + }) + const metascraper = require('metascraper')([require('..')()]) + const metadata = await metascraper({ url: new URL('paper.pdf', origin).href }) + t.is(metadata.title, 'Attention Is All You Need') + t.is(metadata.author, 'Ashish Vaswani') +}) + +test('is a no-op without a PDF url', async t => { + const metascraper = require('metascraper')([require('..')()]) + const metadata = await metascraper({ + url: 'https://example.com', + html: '' + }) + + t.deepEqual(metadata, { + author: null, + date: null, + description: null, + image: null, + lang: null, + logo: null, + publisher: null, + title: null + }) +}) diff --git a/packages/metascraper-pdf/test/snapshots/fixtures.js.md b/packages/metascraper-pdf/test/snapshots/fixtures.js.md new file mode 100644 index 000000000..1fe698c24 --- /dev/null +++ b/packages/metascraper-pdf/test/snapshots/fixtures.js.md @@ -0,0 +1,800 @@ +# Snapshot report for `test/fixtures.js` + +The actual snapshot is saved in `fixtures.js.snap`. + +Generated by [AVA](https://avajs.dev). + +## https://www.nber.org/system/files/working_papers/w28110/w28110.pdf + +> Snapshot 1 + + { + author: 'Bruce Sacerdote', + date: '2020-11-11T15:41:50.000Z', + description: 'We analyze the tone of COVID-19 related English-language news articles written since January 1, 2020. Ninety one percent of stories by U.S. major media outlets are negative in tone versus fifty four percent for non-U.S. major sources and sixty five percent for scientific journals.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nber.org%2Fsystem%2Ffiles%2Fworking_papers%2Fw28110%2Fw28110.pdf&sz=128', + publisher: 'NBER', + title: 'WHY IS ALL COVID-19 NEWS BAD NEWS?', + } + +## https://www.cs.toronto.edu/~hinton/absps/NatureDeepReview.pdf + +> Snapshot 1 + + { + author: 'Yann LeCun', + date: '2015-05-21T13:57:35.000Z', + description: 'achine-learning technology powers many aspects of modern society: from web searches to content filtering on social networks to recommendations on e-commerce websites, and it is increasingly present in consumer products such as cameras and smartphones.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.cs.toronto.edu%2F~hinton%2Fabsps%2FNatureDeepReview.pdf&sz=128', + publisher: 'NATURE', + title: 'Deep learning', + } + +## https://proceedings.neurips.cc/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf + +> Snapshot 1 + + { + author: 'Alex Krizhevsky', + date: '2012-01-01T00:00:00.000Z', + description: 'We trained a large, deep convolutional neural network to classify the 1.2 million high-resolution images in the ImageNet LSVRC-2010 contest into the 1000 different classes.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper%2F2012%2Ffile%2Fc399862d3b9d6b76c8436e924a68c45b-Paper.pdf&sz=128', + publisher: 'NeurIPS', + title: 'ImageNet Classification with Deep Convolutional Neural Networks', + } + +## https://www.stat.berkeley.edu/~breiman/randomforest2001.pdf + +> Snapshot 1 + + { + author: 'Leo Breiman', + date: '2001-12-12T10:48:11.000Z', + description: 'Random forests are a combination of tree predictors such that each tree depends on the values of a random vector sampled independently and with the same distribution for all trees in the forest. The generalization error for forests converges a.s.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.stat.berkeley.edu%2F~breiman%2Frandomforest2001.pdf&sz=128', + publisher: 'Berkeley', + title: 'RANDOM FORESTS', + } + +## https://homes.cs.washington.edu/~pedrod/papers/cacm12.pdf + +> Snapshot 1 + + { + author: null, + date: '2012-09-13T12:46:33.000Z', + description: 'is needed to successfully develop machine learning applications is not readily available in them. As a result, many machine learning projects take much longer than necessary or wind up producing less-than-ideal results. Yet much of this folk knowledge is fairly easy to communicate.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fhomes.cs.washington.edu%2F~pedrod%2Fpapers%2Fcacm12.pdf&sz=128', + publisher: 'communications of the acm', + title: 'a few useful things to Know about machine Learning', + } + +## https://arxiv.org/pdf/1706.03762v7 + +> Snapshot 1 + + { + author: 'Ashish Vaswani', + date: '2017-06-01T00:00:00.000Z', + description: 'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F1706.03762v7&sz=128', + publisher: 'arXiv', + title: 'Attention Is All You Need', + } + +## https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0230416&type=printable + +> Snapshot 1 + + { + author: 'Giovanni Colavizza', + date: '2020-04-21T03:37:32.000Z', + description: 'Efforts to make research results open and reproducible are increasingly reflected by journal policies encouraging or mandating authors to provide data availability statements. As a consequence of this, there has been a strong uptake of data availability statements in recent literature.', + image: 'data:image/png;base64…1810', + lang: 'en', + logo: 'data:image/png;base64…1810', + publisher: 'PLOS ONE', + title: 'The citation advantage of linking publications to research data', + } + +## https://aclanthology.org/N19-1423.pdf + +> Snapshot 1 + + { + author: 'Jacob Devlin', + date: '2019-04-29T17:36:03.000Z', + description: 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Faclanthology.org%2FN19-1423.pdf&sz=128', + publisher: 'Proceedings of NAACL-HLT', + title: 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding', + } + +## https://jmlr.org/papers/volume15/srivastava14a/srivastava14a.pdf + +> Snapshot 1 + + { + author: 'Nitish Srivastava', + date: '2014-07-17T18:58:32.000Z', + description: 'Deep neural nets with a large number of parameters are very powerful machine learning systems. However, overfitting is a serious problem in such networks.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fjmlr.org%2Fpapers%2Fvolume15%2Fsrivastava14a%2Fsrivastava14a.pdf&sz=128', + publisher: 'JMLR', + title: 'Dropout: A Simple Way to Prevent Neural Networks from Overfitting', + } + +## https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0000308&type=printable + +> Snapshot 1 + + { + author: 'Heather A. Piwowar', + date: '2007-03-06T12:02:06.000Z', + description: ', only trial design features such as size and clinical endpoint showed a significant association with citation rate; covariates relating to the data collection and how the data was made available only showed very weak trends.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fjournals.plos.org%2Fplosone%2Farticle%2Ffile%3Fid%3D10.1371%2Fjournal.pone.0000308%26type%3Dprintable&sz=128', + publisher: 'PLoS ONE', + title: 'Sharing Detailed Research Data Is Associated with Increased Citation Rate', + } + +## https://openaccess.thecvf.com/content_CVPR_2020/papers/He_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf + +> Snapshot 1 + + { + author: 'Kaiming He', + date: '2020-01-01T00:00:00.000Z', + description: 'We present Momentum Contrast (MoCo) for unsupervised visual representation learning. From a perspective on contrastive learning [29] as dictionary look-up, we build a dynamic dictionary with a queue and a moving-averaged encoder.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fopenaccess.thecvf.com%2Fcontent_CVPR_2020%2Fpapers%2FHe_Momentum_Contrast_for_Unsupervised_Visual_Representation_Learning_CVPR_2020_paper.pdf&sz=128', + publisher: 'CVF', + title: 'Momentum Contrast for Unsupervised Visual Representation Learning', + } + +## https://www.frontiersin.org/articles/10.3389/fpsyg.2019.00001/pdf + +> Snapshot 1 + + { + author: 'Miguel Clemente', + date: '2019-01-16T13:14:26.000Z', + description: 'The term harassment is often used to refer two contexts, the workplace and school, but not the legal system itself.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.frontiersin.org%2Farticles%2F10.3389%2Ffpsyg.2019.00001%2Fpdf&sz=128', + publisher: 'Frontiers in Psychology', + title: 'Institutional Violence Against Users of the Family Law Courts and the Legal Harassment Scale', + } + +## https://www.nature.com/articles/s41598-023-28020-5.pdf + +> Snapshot 1 + + { + author: 'Loredana Bellantuono', + date: '2023-01-13T11:15:25.000Z', + description: 'The European Quality of Government Index (EQI) measures the perceived level of government quality by European Union citizens, combining surveys on corruption, impartiality and quality of provided services. It is, thus, an index based on individual subjective evaluations.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nature.com%2Farticles%2Fs41598-023-28020-5.pdf&sz=128', + publisher: 'Scientific Reports', + title: 'Detecting the socio-economic drivers of confidence in government with eXplainable Artificial Intelligence', + } + +## https://link.springer.com/content/pdf/10.1186/s13059-020-02007-1.pdf + +> Snapshot 1 + + { + author: 'Qiang Zhang', + date: '2020-04-24T03:36:38.000Z', + description: 'Background: Influenza is a severe respiratory illness that continually threatens global health. It has been widely known that gut microbiota modulates the host response to protect against influenza infection, but mechanistic details remain largely unknown.', + image: 'data:image/png;base64…15142', + lang: 'en', + logo: 'data:image/png;base64…342', + publisher: 'Springer', + title: 'Influenza infection elicits an expansion of gut population of endogenous Bifidobacterium animalis which protects mice against infection', + } + +## https://www.nber.org/system/files/working_papers/w31161/w31161.pdf + +> Snapshot 1 + + { + author: 'Erik Brynjolfsson', + date: '2023-11-03T20:58:21.000Z', + description: 'New AI tools have the potential to change the way workers perform and learn, but little is known about their impacts on the job. In this paper, we study the staggered introduction of a generative AI-based conversational assistant using data from 5,179 customer support agents.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nber.org%2Fsystem%2Ffiles%2Fworking_papers%2Fw31161%2Fw31161.pdf&sz=128', + publisher: 'NBER', + title: 'GENERATIVE AI AT WORK', + } + +## https://cdn.elifesciences.org/articles/00013/elife-00013-v1.pdf + +> Snapshot 1 + + { + author: 'Rosanna A Alegado', + date: '2012-10-05T11:17:37.000Z', + description: 'Bacterially-produced small molecules exert profound influences on animal health, morphogenesis, and evolution through poorly understood mechanisms.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fcdn.elifesciences.org%2Farticles%2F00013%2Felife-00013-v1.pdf&sz=128', + publisher: 'eLife', + title: 'A bacterial sulfonolipid triggers multicellular development in the closest living relatives of animals', + } + +## https://aclanthology.org/2020.acl-main.703.pdf + +> Snapshot 1 + + { + author: 'Mike Lewis', + date: '2020-06-06T22:24:05.000Z', + description: 'We present BART, a denoising autoencoder for pretraining sequence-to-sequence models. BART is trained by (1) corrupting text with an arbitrary noising function, and (2) learning a model to reconstruct the original text.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Faclanthology.org%2F2020.acl-main.703.pdf&sz=128', + publisher: 'Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics', + title: 'BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension', + } + +## https://arxiv.org/pdf/2303.08774v6 + +> Snapshot 1 + + { + author: 'OpenAI', + date: '2023-03-01T00:00:00.000Z', + description: 'We report the development of GPT-4, a large-scale, multimodal model which can accept image and text inputs and produce text outputs.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F2303.08774v6&sz=128', + publisher: 'arXiv', + title: 'GPT-4 Technical Report', + } + +## https://www.redalyc.org/pdf/440/44029444004.pdf + +> Snapshot 1 + + { + author: 'Zayra Elisa Carvajal-Portuguez', + date: '2013-01-01T00:00:00.000Z', + description: 'Enseñanza del inglés en secundaria: una propuesta innovadora. Educación Carvajal-Portuguez, Zayra Elisa vol. 37, núm. 2, julio-diciembre, 2013, pp. 79-101 Universidad de Costa Rica San Pedro, Montes de Oca, Costa Rica', + image: 'data:image/png;base64…2574', + lang: 'es', + logo: 'data:image/png;base64…15010', + publisher: 'Redalyc', + title: 'Enseñanza del inglés en secundaria: una propuesta innovadora', + } + +## https://ceur-ws.org/Vol-2600/paper1.pdf + +> Snapshot 1 + + { + author: 'Sheikh Rabiul Islam', + date: '2020-05-05T11:01:56.000Z', + description: 'Artificial Intelligence (AI) has become an integral part of modern-day security solutions for its ability to learn very complex functions and handling “Big Data”.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fceur-ws.org%2FVol-2600%2Fpaper1.pdf&sz=128', + publisher: 'CEUR-WS', + title: 'Domain Knowledge Aided Explainable Artificial Intelligence for Intrusion Detection and Response', + } + +## https://arxiv.org/pdf/1512.03385v1 + +> Snapshot 1 + + { + author: 'Kaiming He', + date: '2015-12-01T00:00:00.000Z', + description: 'Deeper neural networks are more difficult to train. We present a residual learning framework to ease the training of networks that are substantially deeper than those used previously.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F1512.03385v1&sz=128', + publisher: 'arXiv', + title: 'Deep Residual Learning for Image Recognition', + } + +## https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf + +> Snapshot 1 + + { + author: 'Long Ouyang', + date: '2022-01-01T00:00:00.000Z', + description: 'Making language models bigger does not inherently make them better at following a user’s intent. For example, large language models can generate outputs that are untruthful, toxic, or simply not helpful to the user. In other words, these models are not aligned with their users.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper_files%2Fpaper%2F2022%2Ffile%2Fb1efde53be364a73914f58805a001731-Paper-Conference.pdf&sz=128', + publisher: '36th Conference on Neural Information Processing Systems (NeurIPS )', + title: 'Training language models to follow instructions with human feedback', + } + +## https://journals.plos.org/plosbiology/article/file?id=10.1371/journal.pbio.3000410&type=printable + +> Snapshot 1 + + { + author: 'Nathalie Percie du Sert', + date: '2020-07-11T04:57:33.000Z', + description: 'Reproducible science requires transparent reporting. The ARRIVE guidelines (Animal Research: Reporting of In Vivo Experiments) were originally developed in 2010 to improve the reporting of animal research.', + image: 'data:image/png;base64…1810', + lang: 'en', + logo: 'data:image/png;base64…1810', + publisher: 'PLOS BIOLOGY', + title: 'The ARRIVE guidelines 2.0: Updated guidelines for reporting animal research', + } + +## https://arxiv.org/pdf/2005.14165v4 + +> Snapshot 1 + + { + author: 'Tom B. Brown', + date: '2020-05-01T00:00:00.000Z', + description: 'Recent work has demonstrated substantial gains on many NLP tasks and benchmarks by pre-training on a large corpus of text followed by fine-tuning on a specific task.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F2005.14165v4&sz=128', + publisher: 'arXiv', + title: 'Language Models are Few-Shot Learners', + } + +## https://arxiv.org/pdf/1412.6980v9 + +> Snapshot 1 + + { + author: 'Diederik P. Kingma', + date: '2014-12-01T00:00:00.000Z', + description: 'We introduce Adam, an algorithm for first-order gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F1412.6980v9&sz=128', + publisher: 'arXiv', + title: 'Adam: A Method for Stochastic Optimization', + } + +## https://arxiv.org/pdf/2010.11929v2 + +> Snapshot 1 + + { + author: 'Alexey Dosovitskiy', + date: '2020-10-01T00:00:00.000Z', + description: 'While the Transformer architecture has become the de-facto standard for natural language processing tasks, its applications to computer vision remain limited.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F2010.11929v2&sz=128', + publisher: 'arXiv', + title: 'AN IMAGE IS WORTH 16X16 WORDS: TRANSFORMERS FOR IMAGE RECOGNITION AT SCALE', + } + +## https://arxiv.org/pdf/1810.04805v2 + +> Snapshot 1 + + { + author: 'Jacob Devlin', + date: '2018-10-01T00:00:00.000Z', + description: 'We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F1810.04805v2&sz=128', + publisher: 'arXiv', + title: 'BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding', + } + +## https://arxiv.org/pdf/2203.02155v1 + +> Snapshot 1 + + { + author: 'Long Ouyang', + date: '2022-03-01T00:00:00.000Z', + description: 'Making language models bigger does not inherently make them better at following a user’s intent. For example, large language models can generate outputs that are untruthful, toxic, or simply not helpful to the user. In other words, these models are not aligned with their users.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2F2203.02155v1&sz=128', + publisher: 'arXiv', + title: 'Training language models to follow instructions with human feedback', + } + +## https://arxiv.org/pdf/cs/0102004v1 + +> Snapshot 1 + + { + author: 'Joseph O’Rourke', + date: '2001-02-01T00:00:00.000Z', + description: 'The recent result that n congruent balls in R have at most 4 distinct geometric permutations is described. A line ℓ stabs a set S of geometric objects in R if ℓ intersects every member of S. Such a stabber is often called a line traversal of S.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Farxiv.org%2Fpdf%2Fcs%2F0102004v1&sz=128', + publisher: 'arXiv', + title: 'Computational Geometry Column 41', + } + +## https://arxiv.org/pdf/2104.08663v4 + +> Snapshot 1 + + { + author: 'Nandan Thakur', + date: '2021-04-01T00:00:00.000Z', + description: 'Existing neural information retrieval (IR) models have often been studied in homogeneous and narrow settings, which has considerably limited insights into their out-of-distribution (OOD) generalization capabilities.', + image: 'data:image/png;base64…11398', + lang: 'en', + logo: 'data:image/png;base64…11398', + publisher: 'arXiv', + title: 'BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models', + } + +## https://www.jmlr.org/papers/volume12/pedregosa11a/pedregosa11a.pdf + +> Snapshot 1 + + { + author: 'Fabian Pedregosa', + date: '2011-10-11T19:45:40.000Z', + description: 'Scikit-learn is a Python module integrating a wide range of state-of-the-art machine learning algorithms for medium-scale supervised and unsupervised problems. This package focuses on bringing machine learning to non-specialists using a general-purpose high-level language.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.jmlr.org%2Fpapers%2Fvolume12%2Fpedregosa11a%2Fpedregosa11a.pdf&sz=128', + publisher: 'JMLR', + title: 'Scikit-learn: Machine Learning in Python', + } + +## https://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf + +> Snapshot 1 + + { + author: 'James Bergstra', + date: '2012-02-16T22:46:50.000Z', + description: 'Grid search and manual search are the most widely used strategies for hyper-parameter optimization. This paper shows empirically and theoretically that randomly chosen trials are more efficient for hyper-parameter optimization than trials on a grid.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.jmlr.org%2Fpapers%2Fvolume13%2Fbergstra12a%2Fbergstra12a.pdf&sz=128', + publisher: 'JMLR', + title: 'Random Search for Hyper-Parameter Optimization', + } + +## https://openaccess.thecvf.com/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf + +> Snapshot 1 + + { + author: 'Kaiming He', + date: '2016-01-01T00:00:00.000Z', + description: 'Deeper neural networks are more difficult to train. We present a residual learning framework to ease the training of networks that are substantially deeper than those used previously.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fopenaccess.thecvf.com%2Fcontent_cvpr_2016%2Fpapers%2FHe_Deep_Residual_Learning_CVPR_2016_paper.pdf&sz=128', + publisher: 'CVF', + title: 'Deep Residual Learning for Image Recognition', + } + +## https://openaccess.thecvf.com/content_ICCV_2017/papers/He_Mask_R-CNN_ICCV_2017_paper.pdf + +> Snapshot 1 + + { + author: 'Kaiming He', + date: '2017-01-01T00:00:00.000Z', + description: 'We present a conceptually simple, flexible, and general framework for object instance segmentation. Our approach efficiently detects objects in an image while simultaneously generating a high-quality segmentation mask for each instance.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fopenaccess.thecvf.com%2Fcontent_ICCV_2017%2Fpapers%2FHe_Mask_R-CNN_ICCV_2017_paper.pdf&sz=128', + publisher: 'CVF', + title: 'Mask R-CNN', + } + +## https://aclanthology.org/D19-1410.pdf + +> Snapshot 1 + + { + author: 'Nils Reimers', + date: '2019-08-27T08:41:10.000Z', + description: 'BERT (Devlin et al., 2018) and RoBERTa (Liu et al., 2019) has set a new state-of-the-art performance on sentence-pair regression tasks like semantic textual similarity (STS).', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Faclanthology.org%2FD19-1410.pdf&sz=128', + publisher: 'Proceedings of the Conference on Empirical Methods in Natural Language Processing', + title: 'Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks', + } + +## https://aclanthology.org/P16-1162.pdf + +> Snapshot 1 + + { + author: 'Rico Sennrich', + date: '2016-07-21T02:45:13.000Z', + description: 'Neural machine translation (NMT) models typically operate with a fixed vocabulary, but translation is an open-vocabulary problem. Previous work addresses the translation of out-of-vocabulary words by backing off to a dictionary.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Faclanthology.org%2FP16-1162.pdf&sz=128', + publisher: 'Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics', + title: 'Neural Machine Translation of Rare Words with Subword Units', + } + +## https://proceedings.neurips.cc/paper/2015/file/14bfa6bb14875e45bba028a21ed38046-Paper.pdf + +> Snapshot 1 + + { + author: 'Shaoqing Ren', + date: '2015-01-01T00:00:00.000Z', + description: 'State-of-the-art object detection networks depend on region proposal algorithms to hypothesize object locations. Advances like SPPnet [7] and Fast R-CNN [5] have reduced the running time of these detection networks, exposing region proposal computation as a bottleneck.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fproceedings.neurips.cc%2Fpaper%2F2015%2Ffile%2F14bfa6bb14875e45bba028a21ed38046-Paper.pdf&sz=128', + publisher: 'NeurIPS', + title: 'Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks', + } + +## https://www.nber.org/system/files/working_papers/w26947/w26947.pdf + +> Snapshot 1 + + { + author: 'Titan Alon', + date: '2020-04-01T18:28:10.000Z', + description: 'The economic downturn caused by the current COVID-19 outbreak has substantial implications for gender equality, both during the downturn and the subsequent recovery.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nber.org%2Fsystem%2Ffiles%2Fworking_papers%2Fw26947%2Fw26947.pdf&sz=128', + publisher: 'NBER', + title: 'THE IMPACT OF COVID-19 ON GENDER EQUALITY', + } + +## https://www.nber.org/system/files/working_papers/w30957/w30957.pdf + +> Snapshot 1 + + { + author: 'Anton Korinek', + date: '2023-02-10T15:18:13.000Z', + description: 'Large language models (LLMs) such as ChatGPT have the potential to revolutionize research in economics and other disciplines.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nber.org%2Fsystem%2Ffiles%2Fworking_papers%2Fw30957%2Fw30957.pdf&sz=128', + publisher: 'NBER', + title: 'LANGUAGE MODELS AND COGNITIVE AUTOMATION FOR ECONOMIC RESEARCH', + } + +## https://journals.plos.org/plosmedicine/article/file?id=10.1371/journal.pmed.1003583&type=printable + +> Snapshot 1 + + { + author: 'Matthew J. Page', + date: '2021-03-24T09:31:13.000Z', + description: 'points complete, and accurate account of why the review was done, what they did, and what they found PLOS MEDICINE (UG1EY020522), National Institutes of Health, United States. LAM is supported by a National Institute for Health Research Doctoral Research Fellowship (DRF-2018-11-ST2-048).', + image: 'data:image/png;base64…1810', + lang: 'en', + logo: 'data:image/png;base64…1810', + publisher: 'PLOS Medicine', + title: 'The PRISMA 2020 statement: An updated guideline for reporting systematic reviews', + } + +## https://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1005510&type=printable + +> Snapshot 1 + + { + author: 'Greg Wilson', + date: '2017-06-21T09:08:26.000Z', + description: 'Computers are now essential in all branches of science, but most researchers are never taught the equivalent of basic lab skills for research computing.', + image: 'data:image/png;base64…1070', + lang: 'en', + logo: 'data:image/png;base64…1070', + publisher: 'PLOS Computational Biology', + title: 'Good enough practices in scientific computing', + } + +## https://cdn.elifesciences.org/articles/57443/elife-57443-v2.pdf + +> Snapshot 1 + + { + author: 'Louis K. Scheffer', + date: '2020-09-03T05:24:34.000Z', + description: 'The neural circuits responsible for animal behavior remain largely unknown. We33 summarize new methods and present the circuitry of a large fraction of the brain of the fruit fly34 Drosophila melanogaster.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fcdn.elifesciences.org%2Farticles%2F57443%2Felife-57443-v2.pdf&sz=128', + publisher: 'eLife', + title: 'A Connectome and Analysis of the Adult Drosophila Central Brain', + } + +## https://www.frontiersin.org/articles/10.3389/fnins.2020.00001/pdf + +> Snapshot 1 + + { + author: 'Yu Xie', + date: '2020-01-21T06:02:46.000Z', + description: 'Networks, such as social networks, biochemical networks, and protein-protein interaction networks are ubiquitous in the real world.', + image: 'data:image/png;base64…1610', + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.frontiersin.org%2Farticles%2F10.3389%2Ffnins.2020.00001%2Fpdf&sz=128', + publisher: 'Frontiers in Neuroscience', + title: 'Multi-Task Network Representation Learning', + } + +## https://www.nature.com/articles/s41598-020-69250-1.pdf + +> Snapshot 1 + + { + author: 'Micah J. Sheller', + date: '2020-07-22T07:29:35.000Z', + description: 'Several studies underscore the potential of deep learning in identifying complex patterns, leading to diagnostic and prognostic biomarkers.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nature.com%2Farticles%2Fs41598-020-69250-1.pdf&sz=128', + publisher: 'Scientific RepoRtS', + title: 'Federated learning in medicine: facilitating multi-institutional collaborations without sharing patient data', + } + +## https://www.nature.com/articles/s41467-020-17419-7.pdf + +> Snapshot 1 + + { + author: 'Jonathan G. Richens', + date: '2021-03-30T13:00:12.000Z', + description: 'Machine learning promises to revolutionize clinical decision making and diagnosis. In medical diagnosis a doctor aims to explain a patient’s symptoms by determining the diseases causing them.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.nature.com%2Farticles%2Fs41467-020-17419-7.pdf&sz=128', + publisher: 'NATURE COMMUNICATIONS', + title: 'Improving the accuracy of medical diagnosis with causal machine learning', + } + +## https://link.springer.com/content/pdf/10.1186/s12874-020-01057-0.pdf + +> Snapshot 1 + + { + author: 'Helen Le Sueur', + date: '2020-06-23T14:17:55.000Z', + description: 'Background: Individual clinical trials and cohort studies are a useful source of data, often under-utilised once a study has ended.', + image: 'data:image/png;base64…22214', + lang: 'en', + logo: 'data:image/png;base64…302', + publisher: 'Springer', + title: 'The challenges in data integration – heterogeneity and complexity in clinical trials and patient registries of Systemic Lupus Erythematosus', + } + +## https://ceur-ws.org/Vol-3226/paper1.pdf + +> Snapshot 1 + + { + author: 'Josef Zelinka', + date: '2022-09-08T19:32:41.000Z', + description: 'For autonomous robots operating in an unknown environment, it is important to assess the traversability of the surrounding terrain to improve path planning and decision-making on where to navigate next in a cost-efficient way.', + image: 'data:image/png;base64…1658', + lang: 'en', + logo: 'data:image/png;base64…26950', + publisher: 'CEUR-WS', + title: 'Deep Transfer Learning of Traversability Assessment for Heterogeneous Robots', + } + +## https://www.redalyc.org/pdf/567/56712871011.pdf + +> Snapshot 1 + + { + author: 'Begoña Martínez Domínguez', + date: '2009-01-01T00:00:00.000Z', + description: 'UNA OPORTUNIDAD PARA QUE JÓVENES QUE FRACASAN EN LA ESCUELA PUEDAN SALIR DE LA ZONA DE RIESGO DE EXCLUSIÓN. LA EXPERIENCIA DE LOS CENTROS DE INICIACIÓN PROFESIONAL EN LA COMUNIDAD AUTÓNOMA VASCA. Profesorado. Revista de Currículum y Formación de Profesorado Martínez Domínguez, Begoña; Mendizábal Ituarte, Amaia; Sostoa Gaztelu-Urrutia, Virginia P. vol. 13, núm. 3, 2009, pp. 239-271 Universidad de Granada Granada, España', + image: 'data:image/png;base64…2574', + lang: 'es', + logo: 'data:image/png;base64…21742', + publisher: 'Redalyc', + title: 'UNA OPORTUNIDAD PARA QUE JÓVENES QUE FRACASAN EN LA ESCUELA PUEDAN SALIR DE LA ZONA DE RIESGO DE EXCLUSIÓN. LA EXPERIENCIA DE LOS CENTROS DE INICIACIÓN PROFESIONAL EN LA COMUNIDAD AUTÓNOMA VASCA', + } + +## https://bmcbioinformatics.biomedcentral.com/counter/pdf/10.1186/s12859-020-3418-9.pdf + +> Snapshot 1 + + { + author: 'Lucile Mégret', + date: '2020-02-24T02:45:03.000Z', + description: 'Background: MicroRNA (miRNA) regulation is associated with several diseases, including neurodegenerative diseases. Several approaches can be used for modeling miRNA regulation. However, their precision may be limited for analyzing multidimensional data.', + image: 'data:image/png;base64…34334', + lang: 'en', + logo: 'data:image/png;base64…302', + publisher: 'BMC', + title: 'Combining feature selection and shape analysis uncovers precise rules for miRNA regulation in Huntington’s disease mice', + } + +## https://www.bis.org/publ/work1000.pdf + +> Snapshot 1 + + { + author: 'Valentina Bruno', + date: '2022-02-15T07:11:00.000Z', + description: 'by Valentina Bruno, Ilhyock Shim and Hyun Song Shin February 2022 JEL classification: G12, G15, G23. Keywords: global liquidity, pricing factor, emerging market, exchange rate.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.bis.org%2Fpubl%2Fwork1000.pdf&sz=128', + publisher: 'BIS', + title: 'Dollar beta and stock returns', + } + +## https://bitcoin.org/bitcoin.pdf + +> Snapshot 1 + + { + author: 'Satoshi Nakamoto', + date: '2009-03-24T17:33:15.000Z', + description: 'A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fbitcoin.org%2Fbitcoin.pdf&sz=128', + publisher: 'Bitcoin', + title: 'Bitcoin: A Peer-to-Peer Electronic Cash System', + } + +## https://www.berkshirehathaway.com/letters/2023ltr.pdf + +> Snapshot 1 + + { + author: null, + date: '2024-02-23T23:06:20.000Z', + description: 'Charlie Munger died on November 28, just 33 days before his 100th birthday. Though born and raised in Omaha, he spent 80% of his life domiciled elsewhere. Consequently, it was not until 1959 when he was 35 that I first met him. In 1962, he decided that he should take up money management.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fwww.berkshirehathaway.com%2Fletters%2F2023ltr.pdf&sz=128', + publisher: 'Berkshire Hathaway', + title: 'Charlie Munger – The Architect of Berkshire Hathaway', + } + +## https://cdn.openai.com/papers/gpt-4.pdf + +> Snapshot 1 + + { + author: 'OpenAI', + date: '2023-03-27T17:45:47.000Z', + description: 'We report the development of GPT-4, a large-scale, multimodal model which can accept image and text inputs and produce text outputs.', + image: null, + lang: 'en', + logo: 'https://www.google.com/s2/favicons?domain_url=https%3A%2F%2Fcdn.openai.com%2Fpapers%2Fgpt-4.pdf&sz=128', + publisher: 'OpenAI', + title: 'GPT-4 Technical Report', + } diff --git a/packages/metascraper-pdf/test/snapshots/fixtures.js.snap b/packages/metascraper-pdf/test/snapshots/fixtures.js.snap new file mode 100644 index 000000000..beac95fd4 Binary files /dev/null and b/packages/metascraper-pdf/test/snapshots/fixtures.js.snap differ diff --git a/packages/metascraper-pdf/test/unit.js b/packages/metascraper-pdf/test/unit.js new file mode 100644 index 000000000..d38a0dde0 --- /dev/null +++ b/packages/metascraper-pdf/test/unit.js @@ -0,0 +1,232 @@ +'use strict' + +const test = require('ava').default + +const { identifierDate, documentYear } = require('../src/date') +const { getDescription } = require('../src/description') +const { getTitle, isByline } = require('../src/title') +const { isBannerLine, isPersonName, splitNamePairs } = require('../src/text') +const { + publisherFromLine, + publisherFromUrl, + isVenue +} = require('../src/publisher') +const { readEmbedded, toDate } = require('../src/embedded') +const { nameCount, toAuthor } = require('../src/author') +const { getLang } = require('../src/lang') +const { getMedia } = require('../src/media') + +const line = (index, size, text) => ({ + index, + pageIndex: index, + size, + text, + y: 700 - index * 12 +}) + +test('a heading, a company and a place are not authors', t => { + t.true(isPersonName('Yoshua Bengio')) + t.true(isPersonName('Rosanna A Alegado')) + t.true(isPersonName('Carvajal-Portuguez, Zayra Elisa')) + t.false(isPersonName('RANDOM FORESTS')) + t.false(isPersonName('Google AI Language')) + t.false(isPersonName('Cookeville, U.S')) + t.false(isPersonName('Costa Rica')) + t.false(isPersonName('A Few Useful Things to Know')) +}) + +test('publishers print banners above the title', t => { + t.true(isBannerLine('NBER WORKING PAPER SERIES')) + t.true(isBannerLine('Working Paper 31161')) + t.true(isBannerLine('arXiv:cs/0102004v1 [cs.CG] 6 Feb 2001')) + t.true(isBannerLine('REVIEW')) + t.false(isBannerLine('Deep learning')) +}) + +test('a title wraps, a byline repeats', t => { + const wrapped = [ + line(0, 17, 'AN IMAGE IS WORTH 16X16 WORDS:'), + line(1, 17, 'TRANSFORMERS FOR IMAGE RECOGNITION AT SCALE'), + line(2, 10, 'Alexey Dosovitskiy') + ] + t.is( + getTitle(wrapped).text, + 'AN IMAGE IS WORTH 16X16 WORDS: TRANSFORMERS FOR IMAGE RECOGNITION AT SCALE' + ) + + const cover = [ + line(0, 12, 'NBER WORKING PAPER SERIES'), + line(1, 12, 'GENERATIVE AI AT WORK'), + line(2, 12, 'Erik Brynjolfsson'), + line(3, 12, 'Danielle Li') + ] + t.is(getTitle(cover).text, 'GENERATIVE AI AT WORK') + t.true(isByline(cover[2], cover)) + t.false(isByline(cover[1], cover)) +}) + +test('a byline survives emails, superscripts and editors', t => { + t.is( + toAuthor([line(0, 10, 'Nitish Srivastava nitish@cs.toronto.edu')], [0]), + 'Nitish Srivastava' + ) + t.is( + toAuthor( + [line(0, 10, 'Loredana Bellantuono1,2, Flaviana Palmisano3')], + [0] + ), + 'Loredana Bellantuono, Flaviana Palmisano' + ) + t.is(toAuthor([line(0, 10, 'Editor: Yoshua Bengio')], [0]), null) + t.deepEqual(splitNamePairs('Jacob Devlin Ming-Wei Chang Kenton Lee'), [ + 'Jacob Devlin', + 'Ming-Wei Chang', + 'Kenton Lee' + ]) + t.deepEqual(splitNamePairs('Deep Residual Learning'), []) +}) + +test('a venue is read out of the running header', t => { + t.is( + publisherFromLine( + 'Scientific Reports | (2023) 13:1234 | https://doi.org/10.1038/x' + ), + 'Scientific Reports' + ) + t.is( + publisherFromLine('4 3 6 | N A T U R E | V O L 5 2 1 | 2 8 M A Y 2 0 1 5'), + 'NATURE' + ) + t.is(publisherFromUrl('https://cdn.openai.com/papers/gpt-4.pdf'), 'OpenAI') + t.is( + publisherFromUrl('https://www.berkshirehathaway.com/letters/2023ltr.pdf'), + 'Berkshire Hathaway' + ) + t.is(publisherFromUrl('https://bitcoin.org/bitcoin.pdf'), 'Bitcoin') + t.is(publisherFromUrl('https://arxiv.org/pdf/1706.03762v7'), 'arXiv') + t.is(publisherFromUrl('https://www.nber.org/papers/w31161.pdf'), 'NBER') + t.true( + isVenue('Frontiers in Psychology', 'https://www.frontiersin.org/x/pdf') + ) + t.false(isVenue('official NBER publications', 'https://www.nber.org/x.pdf')) +}) + +test('the identifier dates the document', t => { + t.is( + identifierDate('https://arxiv.org/pdf/1706.03762v7'), + '2017-06-01T00:00:00.000Z' + ) + t.is( + identifierDate('https://arxiv.org/pdf/cs/0102004v1'), + '2001-02-01T00:00:00.000Z' + ) + t.is( + identifierDate('https://proceedings.neurips.cc/paper/2012/file/x.pdf'), + '2012-01-01T00:00:00.000Z' + ) + t.is(identifierDate('https://bitcoin.org/bitcoin.pdf'), null) + t.is( + documentYear( + [line(0, 10, 'Published as a conference paper at ICLR 2021')], + { now: new Date('2026-01-01') } + ), + 2021 + ) +}) + +test('the description is the abstract, never the running header', t => { + const lines = [ + line(0, 9, 'Frontiers in Psychology | Volume 10 | Article 1'), + line(1, 14, 'Institutional Violence'), + line(2, 10, 'Abstract'), + line( + 3, + 10, + 'This work analyses the psychological consequences of institutional violence in family law courts.' + ), + line(4, 10, '1 Introduction'), + line(5, 10, 'Funding was provided by the ministry of science.') + ] + + t.true(getDescription(lines).startsWith('This work analyses')) + t.false(getDescription(lines).includes('Funding was provided')) +}) + +test('an inverted name is one author', t => { + t.is(nameCount('Doe, Jane'), 1) + t.is(nameCount('Jane Doe, John Smith'), 2) + t.is(nameCount('Jane Doe AND John Smith'), 2) + t.is(nameCount('Doe, Jane; Smith, John'), 2) +}) + +test('a PDF date keeps its offset and rejects year zero', t => { + t.is(toDate("D:20201111104150-05'00'"), '2020-11-11T15:41:50.000Z') + t.is(toDate('D:20201111104150Z'), '2020-11-11T10:41:50.000Z') + t.is(toDate('D:20201111104150'), '2020-11-11T10:41:50.000Z') + t.is(toDate('D:00000000000000'), null) + t.is(toDate('D:20240230'), null) +}) + +test('generator noise never reaches a property', t => { + const embedded = readEmbedded({ + info: { + Title: 'Microsoft Word - final draft.docx', + Author: 'Windows User', + Subject: '2017 IEEE International Conference on Computer Vision', + Creator: 'Acrobat' + } + }) + + t.is(embedded.title, null) + t.is(embedded.author, null) + t.is(embedded.description, null) +}) + +test('a journal name plus a doi is not a description', t => { + const embedded = readEmbedded({ + info: { + Subject: 'Genome Biology, 2020, doi:10.1186/s13059-020-02007-1' + } + }) + t.is(embedded.description, null) +}) + +test('lang comes from the host or the words on the page', t => { + t.is( + getLang( + 'We study the staggered introduction of a generative AI assistant', + { + url: 'https://example.com/paper.pdf' + } + ), + 'en' + ) + t.is( + getLang( + 'Enseñanza del inglés en secundaria para los estudiantes de la escuela', + { + url: 'https://www.redalyc.org/pdf/x.pdf' + } + ), + 'es' + ) +}) + +test('a small square image is the logo; otherwise the host favicon', t => { + const pixels = Buffer.alloc(16 * 16 * 3, 80) + const { image, logo } = getMedia( + [{ width: 16, height: 16, channels: 3, data: pixels }], + { url: 'https://journals.plos.org/article.pdf' } + ) + t.true(image.startsWith('data:image/png;base64,')) + t.true(logo.startsWith('data:image/png;base64,')) + + const fallback = getMedia([], { url: 'https://arxiv.org/pdf/x' }) + t.is(fallback.image, null) + t.is( + fallback.logo, + `https://www.google.com/s2/favicons?domain_url=${encodeURIComponent( + 'https://arxiv.org/pdf/x' + )}&sz=128` + ) +})