From 1f5b4449896e9105b4b1743ff27882d7151df3c0 Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:22 +0200 Subject: [PATCH 01/10] Add uDSV harness benchmarks and a README benchmark section benchmarks/udsv/sweep.ts drives uDSV's bench/runone.cjs unmodified over spliterator and the parsers it ships adapters for, measures peak RSS through GNU time, and writes results.json; render.ts prints Markdown tables with throughput and memory columns. The README section is generated from it and kept honest, losses included. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- README.md | 86 ++++++ benchmarks/udsv/adapters/papaparse-count.ts | 40 +++ benchmarks/udsv/adapters/spliterator-count.ts | 32 +++ .../udsv/adapters/spliterator-memory.ts | 19 ++ .../udsv/adapters/spliterator-stream.ts | 17 ++ benchmarks/udsv/adapters/udsv-count.ts | 53 ++++ benchmarks/udsv/render.ts | 111 +++++++ benchmarks/udsv/results.json | 272 ++++++++++++++++++ benchmarks/udsv/sweep.ts | 233 +++++++++++++++ test/benchmarks/udsv-render.test.ts | 65 +++++ 10 files changed, 928 insertions(+) create mode 100644 benchmarks/udsv/adapters/papaparse-count.ts create mode 100644 benchmarks/udsv/adapters/spliterator-count.ts create mode 100644 benchmarks/udsv/adapters/spliterator-memory.ts create mode 100644 benchmarks/udsv/adapters/spliterator-stream.ts create mode 100644 benchmarks/udsv/adapters/udsv-count.ts create mode 100644 benchmarks/udsv/render.ts create mode 100644 benchmarks/udsv/results.json create mode 100644 benchmarks/udsv/sweep.ts create mode 100644 test/benchmarks/udsv-render.test.ts diff --git a/README.md b/README.md index dbffa08..f195090 100644 --- a/README.md +++ b/README.md @@ -374,6 +374,92 @@ Two things follow from workers being reused, both of them the point rather than A pool smaller than `concurrency` bounds the real parallelism — segments queue for a worker instead of running at once, and `parallelMapWorkers` clamps to the pool's size. Dispose it when you are done, or bind it with `await using` as above. +## Benchmarks + +Numbers from [uDSV's benchmark harness](https://github.com/leeoniya/uDSV/tree/main/bench), run unmodified: its +runner, its adapters for the other parsers, and its timing metric (the geometric mean of parse time over +three-second cycles, reported as MiB/s). Every parser produces arrays of strings, with no typing, trimming, or header +handling. The harness gives each parser a string already in memory; the streaming tables read the file with each +parser's own streaming API instead, retaining every row in the `(stream)` rows and discarding them in the +`(stream, count)` rows. + +Memory is the process's peak resident set over the whole run, less the baseline the runner reports before loading +the parser. An in-memory run therefore includes the input string, which every parser is handed alike, and a retained +run includes the result; the count rows show a parser's own working set. Bars are relative to the largest value in +their column. + +| Date | Hardware | Node | +| ------- | --------------------------------------- | ------- | +| 2026-10 | AMD Ryzen 9 8945HS, 29 GB RAM, NVMe SSD | v26.2.0 | + +`litmus_quoted.csv` is uDSV's generated 20-column file, every cell quoted. The other two are public datasets: +[CMS Open Payments](https://openpaymentsdata.cms.gov/) covered-recipient profiles (386 MiB, 32 columns, quoted) and +address tuples derived from +[HM Land Registry Price Paid](https://www.gov.uk/government/statistical-data-sets/price-paid-data-downloads) +(1.4 GiB, 6 columns, quoted), which only streams because it is past V8's string limit. A parser whose row count +disagreed with uDSV's by more than the harness tolerates is listed with that error rather than timed. + +**litmus_quoted.csv (1.8 MiB, 10K rows), in-memory string** + +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| ----------------- | -------------------------------------------- | -------------------------------------------- | +| csv-simple-parser | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 390 | ░░░░░░░░░░░░ 58 | +| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 370 | ░░░░░░░░░░░░░░░░░ 85 | +| spliterator | ░░░░░░░░░░░░░░░░░░░░░ 209 | ░░░░░░░░░░░░░░░░░░░░░░░░░ 122 | +| but-csv | ░░░░░░░░░░░░░░░░░░░ 187 | ░░░░░░░░░░░░░░░░░░░░░░░ 112 | +| d3-dsv | ░░░░░░░░░░ 96.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | +| PapaParse | ░░░░░░░░ 74.2 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 192 | + +**litmus_quoted.csv (1.8 MiB, 10K rows), streamed from file** + +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| --------------------------- | -------------------------------------------- | -------------------------------------------- | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 261 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 153 | +| uDSV (stream) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 229 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 198 | +| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░ 168 | ░░░░░░░░░░░░░░░░░░░ 93 | +| spliterator (stream) | ░░░░░░░░░░░░░░░░░░░░░░░ 153 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 179 | +| PapaParse (stream, count) | ░░░░░░░░░░░░░░ 89.4 | ░░░░░░░░░░░░░░ 71 | +| PapaParse (stream) | ░░░░░░░░░░ 66.8 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 189 | + +**openpayments_covered-recipient-profile_20260603.csv (386 MiB, 1.7M rows), in-memory string** + +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| ----------------- | --------------------------------------------------- | ---------------------------------------------- | +| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 142 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.81K | +| d3-dsv | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 101 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.85K | +| spliterator | ░░░░░░░░░░░░░░░░░░░░░ 72.9 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.01K | +| PapaParse | Wrong row count! Expected: 1697026, Actual: 1697027 | | +| csv-simple-parser | Wrong row count! Expected: 1697026, Actual: 1697027 | | +| but-csv | Wrong row count! Expected: 1697026, Actual: 1697027 | | + +**openpayments_covered-recipient-profile_20260603.csv (386 MiB, 1.7M rows), streamed from file** + +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| --------------------------- | -------------------------------------------- | ---------------------------------------------- | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 307 | ░░ 147 | +| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░ 205 | ░ 98 | +| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░ 184 | ░ 81 | +| spliterator (stream) | ░░░░░░░░░░░░░ 99 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.37K | +| PapaParse (stream) | ░░░░░░░░░░░ 86.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.71K | +| uDSV (stream) | ░░░░░░░░░░ 75.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.67K | + +**gb-tuples.csv (1.42 GiB), streamed from file** + +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| --------------------------- | -------------------------------------------- | -------------------------------------------- | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 269 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 132 | +| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░ 137 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 101 | +| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░ 118 | ░░░░░░░░░░░░░░░░░░░░░░░ 76 | + +CSV into arrays of strings is parse-bound, and on that work spliterator is mid-pack: uDSV generates a parser per +schema and slices one decoded string, and nothing byte-oriented will catch that. What this harness cannot show is +what spliterator is for — raw byte-range scanning at SIMD speed, a bounded footprint on a file of any size, and the +chainable sequence over the rows. See [Choosing a primitive](#choosing-a-primitive) for where each of those pays. + +To rerun: `benchmarks/udsv/sweep.ts` drives the harness from a sibling uDSV checkout and writes +`benchmarks/udsv/results.json`; `benchmarks/udsv/render.ts` prints the tables above from it. Setup is in the header +of the sweep script. + ## Under the hood ### SIMD acceleration diff --git a/benchmarks/udsv/adapters/papaparse-count.ts b/benchmarks/udsv/adapters/papaparse-count.ts new file mode 100644 index 0000000..e32de9b --- /dev/null +++ b/benchmarks/udsv/adapters/papaparse-count.ts @@ -0,0 +1,40 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: PapaParse streaming a file and counting rows and cells. The counterpart of `spliterator-count.ts`. + */ + +import { createReadStream } from "node:fs" +import { createRequire } from "node:module" + +export const name = "PapaParse (stream, count)" +export const repo = "https://github.com/mholt/PapaParse" + +interface Papa { + parse( + stream: NodeJS.ReadableStream, + options: { step(result: { data: unknown[] }): void; complete(): void; error(error: Error): void } + ): void +} + +export async function load() { + const require = createRequire(process.env.UDSV_ROOT + "/bench/") + const Papa = require("papaparse") as Papa + + return (_csvStr: string, path: string) => + new Promise((resolve, reject) => { + let rows = 0 + let cells = 0 + + Papa.parse(createReadStream(path), { + step: (result) => { + rows++ + cells += result.data.length + }, + complete: () => resolve([[rows], [cells]]), + error: reject, + }) + }) +} diff --git a/benchmarks/udsv/adapters/spliterator-count.ts b/benchmarks/udsv/adapters/spliterator-count.ts new file mode 100644 index 0000000..ba50c47 --- /dev/null +++ b/benchmarks/udsv/adapters/spliterator-count.ts @@ -0,0 +1,32 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: stream a file and count rows and cells without retaining them. uDSV's own non-retained adapters + * sum one numeric column, which the datasets here do not all have, so the count adapters below are the comparison. + */ + +import { CSVSpliterator } from "spliterator" + +export const name = "spliterator (stream, count)" +export const repo = "https://github.com/sister-software/spliterator" + +export async function load() { + return async (_csvStr: string, path: string) => { + let rows = 0 + let cells = 0 + + for await (const row of CSVSpliterator.fromAsync(path, { + mode: "array", + header: false, + trim: false, + bulkThreshold: 0, + })) { + rows++ + cells += row.length + } + + return [[rows], [cells]] + } +} diff --git a/benchmarks/udsv/adapters/spliterator-memory.ts b/benchmarks/udsv/adapters/spliterator-memory.ts new file mode 100644 index 0000000..d373adb --- /dev/null +++ b/benchmarks/udsv/adapters/spliterator-memory.ts @@ -0,0 +1,19 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: parse an in-memory string into string arrays. Mirrors `non-streaming/untyped/uDSV.cjs` there. + */ + +import { CharacterSequence, CSVSpliterator } from "spliterator" + +export const name = "spliterator" +export const repo = "https://github.com/sister-software/spliterator" + +export async function load() { + await CharacterSequence.whenReady() + + return (csvStr: string) => + Promise.resolve(CSVSpliterator.from(csvStr, { mode: "array", header: false, trim: false }).toArray()) +} diff --git a/benchmarks/udsv/adapters/spliterator-stream.ts b/benchmarks/udsv/adapters/spliterator-stream.ts new file mode 100644 index 0000000..eefdeb5 --- /dev/null +++ b/benchmarks/udsv/adapters/spliterator-stream.ts @@ -0,0 +1,17 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: stream a file into retained string arrays. Mirrors `streaming/untyped/retained/uDSV.cjs` there. + */ + +import { CSVSpliterator } from "spliterator" + +export const name = "spliterator (stream)" +export const repo = "https://github.com/sister-software/spliterator" + +export async function load() { + return (_csvStr: string, path: string) => + CSVSpliterator.fromAsync(path, { mode: "array", header: false, trim: false, bulkThreshold: 0 }).toArray() +} diff --git a/benchmarks/udsv/adapters/udsv-count.ts b/benchmarks/udsv/adapters/udsv-count.ts new file mode 100644 index 0000000..6290d66 --- /dev/null +++ b/benchmarks/udsv/adapters/udsv-count.ts @@ -0,0 +1,53 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: uDSV streaming a file and counting rows and cells. The counterpart of `spliterator-count.ts`. + */ + +import { createReadStream } from "node:fs" +import { createRequire } from "node:module" + +export const name = "uDSV (stream, count)" +export const repo = "https://github.com/leeoniya/uDSV" + +interface UDSVParser { + stringArrs: unknown + chunk: (chunk: string, mode: unknown, each: (row: string[]) => void) => void + end: () => void +} + +interface UDSV { + inferSchema: (chunk: string) => unknown + initParser: (schema: unknown) => UDSVParser +} + +export async function load() { + const require = createRequire(process.env.UDSV_ROOT + "/") + const { inferSchema, initParser } = require("./dist/uDSV.cjs.js") as UDSV + + return (_csvStr: string, path: string) => + new Promise((resolve) => { + const stream = createReadStream(path, { highWaterMark: 1024 * 1024 }) + let parser: UDSVParser | null = null + let rows = 0 + let cells = 0 + + stream.on("data", (chunk) => { + const text = chunk.toString() + + parser ??= initParser(inferSchema(text)) + + parser.chunk(text, parser.stringArrs, (row) => { + rows++ + cells += row.length + }) + }) + + stream.on("end", () => { + parser?.end() + resolve([[rows], [cells]]) + }) + }) +} diff --git a/benchmarks/udsv/render.ts b/benchmarks/udsv/render.ts new file mode 100644 index 0000000..82be495 --- /dev/null +++ b/benchmarks/udsv/render.ts @@ -0,0 +1,111 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * Render `results.json` from `sweep.ts` as Markdown tables with the text bars uDSV's README uses, one table per + * dataset and mode, ready to paste into README.md. + * + * Usage: node out/benchmarks/udsv/render.js [results.json] + */ + +import { readFileSync } from "node:fs" + +/** + * One parser's run over one dataset, as `sweep.ts` records it. Timing fields are `null` when the harness reported an + * error instead of timing the parser. + */ +export interface SweepResult { + data: string + mib: number + name: string + gmean_ms: number | null + mibs: number | null + rss_mib: number | null + rows: number | null + error: string | null +} + +/** + * Which of the harness's two modes a result came from: parsing a string already in memory, or streaming the file. The + * streaming adapters carry "stream" in their name, as uDSV's own do. + */ +export function modeOf(result: SweepResult): "memory" | "stream" { + return result.name.includes("(stream") ? "stream" : "memory" +} + +const BAR_WIDTH = 40 + +const compact = new Intl.NumberFormat("en-US", { notation: "compact", maximumSignificantDigits: 3 }) +const formatNumber = (value: number) => compact.format(value) + +function bar(value: number, max: number): string { + const blocks = max > 0 ? Math.round((value / max) * BAR_WIDTH) : 0 + + return "░".repeat(blocks) + " " + formatNumber(value) +} + +function table(caption: string, columns: string[], rows: string[][]): string { + const widths = columns.map((column, i) => Math.max(column.length, ...rows.map((row) => row[i]!.length))) + const line = (cells: string[]) => "| " + cells.map((cell, i) => cell.padEnd(widths[i]!)).join(" | ") + " |" + + return [ + `**${caption}**`, + "", + line(columns), + "| " + widths.map((width) => "-".repeat(width)).join(" | ") + " |", + ...rows.map(line), + ].join("\n") +} + +/** + * One table for the given results, which must share a dataset and a mode. Errored parsers are listed last with the + * error in place of their bars; timed parsers are sorted fastest first. Bars scale to the largest value in each column, + * so the memory bars read relative to the hungriest parser on that table. + */ +export function renderTable(results: SweepResult[]): string { + const first = results[0] + + if (!first) throw new TypeError("renderTable needs at least one result") + + const timed = results.filter((r) => r.mibs !== null).toSorted((a, b) => b.mibs! - a.mibs!) + const failed = results.filter((r) => r.mibs === null) + const maxMibs = Math.max(0, ...timed.map((r) => r.mibs!)) + const maxRss = Math.max(0, ...timed.map((r) => r.rss_mib ?? 0)) + const rowCount = timed.find((r) => r.rows !== null && r.rows > 2)?.rows + + const mode = modeOf(first) === "memory" ? "in-memory string" : "streamed from file" + const size = first.mib >= 1024 ? `${(first.mib / 1024).toFixed(2)} GiB` : `${formatNumber(first.mib)} MiB` + const header = `${first.data} (${size}${rowCount ? `, ${formatNumber(rowCount)} rows` : ""}), ${mode}` + + const rows = [ + ...timed.map((r) => [r.name, bar(r.mibs!, maxMibs), bar(r.rss_mib ?? 0, maxRss)]), + ...failed.map((r) => [r.name, r.error ?? "error", ""]), + ] + + return table(header, ["Name", "Throughput (MiB/s)", "Peak RSS above baseline (MiB)"], rows) +} + +/** + * One table per dataset and mode, in first-seen order, separated by a blank line. + */ +export function renderTables(results: SweepResult[]): string { + const groups = new Map() + + for (const result of results) { + const key = result.data + "\0" + modeOf(result) + const group = groups.get(key) ?? [] + + group.push(result) + groups.set(key, group) + } + + return [...groups.values()].map(renderTable).join("\n\n") +} + +if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split("/").pop()!)) { + const path = process.argv[2] ?? new URL("../../../benchmarks/udsv/results.json", import.meta.url).pathname + const results = JSON.parse(readFileSync(path, "utf8")) as SweepResult[] + + console.log(renderTables(results)) +} diff --git a/benchmarks/udsv/results.json b/benchmarks/udsv/results.json new file mode 100644 index 0000000..bbf57dd --- /dev/null +++ b/benchmarks/udsv/results.json @@ -0,0 +1,272 @@ +[ + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "spliterator", + "gmean_ms": 8.8, + "mibs": 209.1, + "rss_mib": 122, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "uDSV", + "gmean_ms": 5, + "mibs": 369.6, + "rss_mib": 85, + "rows": 9999, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "PapaParse", + "gmean_ms": 24.9, + "mibs": 74.2, + "rss_mib": 192, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "csv-simple-parser", + "gmean_ms": 4.7, + "mibs": 390.3, + "rss_mib": 58, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "d3-dsv", + "gmean_ms": 19.2, + "mibs": 96.1, + "rss_mib": 196, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "but-csv", + "gmean_ms": 9.9, + "mibs": 187.3, + "rss_mib": 112, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "spliterator (stream)", + "gmean_ms": 12.1, + "mibs": 152.9, + "rss_mib": 179, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "uDSV (stream)", + "gmean_ms": 8.1, + "mibs": 228.8, + "rss_mib": 198, + "rows": 9999, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "PapaParse (stream)", + "gmean_ms": 27.6, + "mibs": 66.8, + "rss_mib": 189, + "rows": 10000, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "spliterator (stream, count)", + "gmean_ms": 11, + "mibs": 168.4, + "rss_mib": 93, + "rows": 2, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "uDSV (stream, count)", + "gmean_ms": 7.1, + "mibs": 261.3, + "rss_mib": 153, + "rows": 2, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "PapaParse (stream, count)", + "gmean_ms": 20.7, + "mibs": 89.4, + "rss_mib": 71, + "rows": 2, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "spliterator", + "gmean_ms": 5292.5, + "mibs": 72.9, + "rss_mib": 3005, + "rows": 1697026, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "uDSV", + "gmean_ms": 2726, + "mibs": 141.6, + "rss_mib": 3814, + "rows": 1697025, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "PapaParse", + "gmean_ms": null, + "mibs": null, + "rss_mib": null, + "rows": null, + "error": "Wrong row count! Expected: 1697026, Actual: 1697027" + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "csv-simple-parser", + "gmean_ms": null, + "mibs": null, + "rss_mib": null, + "rows": null, + "error": "Wrong row count! Expected: 1697026, Actual: 1697027" + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "d3-dsv", + "gmean_ms": 3814.3, + "mibs": 101.2, + "rss_mib": 2852, + "rows": 1697026, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "but-csv", + "gmean_ms": null, + "mibs": null, + "rss_mib": null, + "rows": null, + "error": "Wrong row count! Expected: 1697026, Actual: 1697027" + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "spliterator (stream)", + "gmean_ms": 3901.6, + "mibs": 99, + "rss_mib": 3373, + "rows": 1697026, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "uDSV (stream)", + "gmean_ms": 5137.7, + "mibs": 75.1, + "rss_mib": 3666, + "rows": 1697025, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "PapaParse (stream)", + "gmean_ms": 4484.2, + "mibs": 86.1, + "rss_mib": 3708, + "rows": 1697026, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "spliterator (stream, count)", + "gmean_ms": 1885.5, + "mibs": 204.8, + "rss_mib": 98, + "rows": 2, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "uDSV (stream, count)", + "gmean_ms": 1256.2, + "mibs": 307.3, + "rss_mib": 147, + "rows": 2, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "PapaParse (stream, count)", + "gmean_ms": 2103.5, + "mibs": 183.5, + "rss_mib": 81, + "rows": 2, + "error": null + }, + { + "data": "gb-tuples.csv", + "mib": 1454.2, + "name": "spliterator (stream, count)", + "gmean_ms": 10587.1, + "mibs": 137.4, + "rss_mib": 101, + "rows": 2, + "error": null + }, + { + "data": "gb-tuples.csv", + "mib": 1454.2, + "name": "uDSV (stream, count)", + "gmean_ms": 5413.6, + "mibs": 268.6, + "rss_mib": 132, + "rows": 2, + "error": null + }, + { + "data": "gb-tuples.csv", + "mib": 1454.2, + "name": "PapaParse (stream, count)", + "gmean_ms": 12378.5, + "mibs": 117.5, + "rss_mib": 76, + "rows": 2, + "error": null + } +] diff --git a/benchmarks/udsv/sweep.ts b/benchmarks/udsv/sweep.ts new file mode 100644 index 0000000..8e32bab --- /dev/null +++ b/benchmarks/udsv/sweep.ts @@ -0,0 +1,233 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * Run uDSV's benchmark harness over spliterator and a few of the parsers it ships adapters for, and write the + * results to `benchmarks/udsv/results.json` for `render.ts`. + * + * The harness, its competitor adapters, and its timing metric (geometric mean over three-second cycles, MiB/s) are + * uDSV's own and run unmodified; this script only spawns `bench/runone.cjs` and collects its JSON. One generated + * copy of that runner skips reading the file into a string so the streaming count adapters can take a file past + * V8's string limit. + * + * Memory is measured here rather than taken from the harness, whose figure is the largest RSS step between timing + * cycles and so reads zero for a parser that allocates its working set on the first parse. Each run goes through + * GNU `time`, and the child's peak RSS less the baseline the runner reports before loading the parser is recorded. + * For an in-memory run that includes the input string, which every parser is handed alike; for a count run it is + * the parser's own footprint. + * + * Setup: + * + * git clone https://github.com/leeoniya/uDSV ../uDSV + * (cd ../uDSV && npm install && npm run build && cd bench && npm install && node litmus_gen.cjs) + * yarn compile + * + * Usage: node out/benchmarks/udsv/sweep.js [--udsv=../uDSV] [--memory] [--stream] ... + * + * `--memory` runs the in-memory string parsers, `--stream` the file-streaming ones; both by default. A dataset + * without an entry in uDSV's `bench/expected.json` gets one computed by uDSV itself when it fits in a string, and + * runs with verification off when it does not. Dataset paths are resolved from the current directory and linked into + * the harness's data directory, because the harness keys expectations on the file name. Results merge into + * `results.json` per dataset, so a later sweep over one file refreshes only its tables. + */ + +import { spawnSync } from "node:child_process" +import { existsSync, mkdtempSync, readFileSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs" +import { createRequire } from "node:module" +import { tmpdir } from "node:os" +import { basename, join, resolve } from "node:path" +import { parseArgs } from "node:util" + +import type { SweepResult } from "./render.ts" + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + udsv: { type: "string", default: "../uDSV" }, + memory: { type: "boolean", default: false }, + stream: { type: "boolean", default: false }, + }, +}) + +const udsvRoot = resolve(values.udsv) +const bench = resolve(udsvRoot, "bench") +const adapters = new URL("./adapters/", import.meta.url).pathname +const runMemory = values.memory || !values.stream +const runStream = values.stream || !values.memory + +if (!existsSync(resolve(bench, "runone.cjs"))) { + console.error(`No uDSV bench at ${bench}. See the setup notes at the top of this script.`) + + process.exit(1) +} + +if (!positionals.length) { + console.error("Usage: node out/benchmarks/udsv/sweep.js [--udsv=../uDSV] [--memory] [--stream] ...") + + process.exit(1) +} + +/** + * V8 refuses strings above this many bytes, so larger files only stream. + */ +const STRING_LIMIT = 512 * 1024 * 1024 + +const memoryParsers = [ + adapters + "spliterator-memory.js", + "./non-streaming/untyped/uDSV.cjs", + "./non-streaming/untyped/PapaParse.cjs", + "./non-streaming/untyped/csv-simple-parser.cjs", + "./non-streaming/untyped/d3-dsv.cjs", + "./non-streaming/untyped/but-csv.cjs", +] + +const retainedParsers = [ + adapters + "spliterator-stream.js", + "./streaming/untyped/retained/uDSV.cjs", + "./streaming/untyped/retained/PapaParse.cjs", +] + +const countParsers = [adapters + "spliterator-count.js", adapters + "udsv-count.js", adapters + "papaparse-count.js"] + +// The runner reads the whole file into a string unless the adapter path says otherwise. The count adapters never +// need it, and the largest files cannot provide it, so a generated copy of the runner widens that test. +const peakFile = join(mkdtempSync(join(tmpdir(), "udsv-sweep-")), "peak") + +const runner = readFileSync(resolve(bench, "runone.cjs"), "utf8") +const streamingRunner = resolve(bench, "runone-streaming.generated.cjs") + +writeFileSync( + streamingRunner, + runner.replace("!parserMod.includes('/streaming/non-retained')", "!/non-retained|-count\\.js$/.test(parserMod)") +) + +const expectedPath = resolve(bench, "expected.json") +const expected = JSON.parse(readFileSync(expectedPath, "utf8")) as Record +const require = createRequire(udsvRoot + "/") + +interface UDSV { + inferSchema: (chunk: string) => unknown + initParser: (schema: unknown) => { stringArrs: (text: string) => string[][] } +} + +const datasets = positionals.map((argument) => { + const source = resolve(argument) + const name = basename(source) + const link = resolve(bench, "data", name) + const size = statSync(source).size + + // A dataset already in the harness's data directory, such as the generated litmus files, needs no link. + if (source !== link) { + if (existsSync(link)) { + unlinkSync(link) + } + + symlinkSync(source, link) + } + + if (!expected[name]) { + if (size <= STRING_LIMIT) { + const { inferSchema, initParser } = require("./dist/uDSV.cjs.js") as UDSV + const text = readFileSync(source, "utf8") + const rows = initParser(inferSchema(text)).stringArrs(text) + + // uDSV drops the header row and the harness accepts a count up to two below the expectation, so an + // expectation one above uDSV's count admits both it and parsers that keep the header. + expected[name] = { rows: rows.length + 1, cols: rows[0]!.length } + } else { + expected[name] = { rows: 0, cols: 0 } + } + + writeFileSync(expectedPath, JSON.stringify(expected, null, 2)) + } + + return { name, size, fitsString: size <= STRING_LIMIT } +}) + +const results: SweepResult[] = [] + +function run(dataset: (typeof datasets)[number], parser: string, verify: boolean) { + const counting = countParsers.includes(parser) + + const result = spawnSync( + "/usr/bin/time", + [ + "-f", + "%M", + "-o", + peakFile, + process.execPath, + "--max-old-space-size=20000", + counting ? streamingRunner : "./bench/runone.cjs", + `--data=./bench/data/${dataset.name}`, + `--parser=${parser}`, + `--verify=${verify ? 1 : 0}`, + ], + { cwd: udsvRoot, env: { ...process.env, UDSV_ROOT: udsvRoot }, maxBuffer: 1 << 28 } + ) + + let parsed: { gmean: number | null; rssBase: number | null; rows: number | null; error: string | null } + + try { + parsed = JSON.parse(result.stdout.toString().trim()) + } catch { + parsed = { gmean: null, rssBase: null, rows: null, error: "crash: " + result.stderr.toString().slice(-200).trim() } + } + + // GNU time reports the peak resident set in KiB. + const peakKiB = Number(readFileSync(peakFile, "utf8").trim().split("\n").pop()) + + const peakAboveBaseline = + parsed.gmean === null || parsed.rssBase === null ? null : peakKiB / 1024 - parsed.rssBase / 1_048_576 + + const name = (require(parser.startsWith("/") ? parser : resolve(bench, parser)) as { name: string }).name + const mib = dataset.size / 1_048_576 + + const row: SweepResult = { + data: dataset.name, + mib: +mib.toFixed(1), + name, + gmean_ms: parsed.gmean === null ? null : +parsed.gmean.toFixed(1), + mibs: parsed.gmean === null ? null : +(mib / (parsed.gmean / 1e3)).toFixed(1), + rss_mib: peakAboveBaseline === null ? null : +peakAboveBaseline.toFixed(0), + rows: parsed.rows, + error: parsed.error, + } + + results.push(row) + + console.log(JSON.stringify(row)) +} + +for (const dataset of datasets) { + if (runMemory && dataset.fitsString) { + for (const parser of memoryParsers) { + run(dataset, parser, true) + } + } + + if (runStream) { + if (dataset.fitsString) { + for (const parser of retainedParsers) { + run(dataset, parser, true) + } + } + + for (const parser of countParsers) { + run(dataset, parser, false) + } + } +} + +// Results merge per dataset, so a sweep over one file refreshes its tables and leaves the rest alone. +const out = new URL("../../../benchmarks/udsv/results.json", import.meta.url).pathname +const swept = new Set(datasets.map((dataset) => dataset.name)) + +const kept = existsSync(out) + ? (JSON.parse(readFileSync(out, "utf8")) as SweepResult[]).filter((result) => !swept.has(result.data)) + : [] + +writeFileSync(out, JSON.stringify([...kept, ...results], null, 2) + "\n") + +console.log(`Wrote ${results.length} results to ${out}`) diff --git a/test/benchmarks/udsv-render.test.ts b/test/benchmarks/udsv-render.test.ts new file mode 100644 index 0000000..f0f3351 --- /dev/null +++ b/test/benchmarks/udsv-render.test.ts @@ -0,0 +1,65 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { test } from "vitest" + +import { renderTable, renderTables, type SweepResult } from "../../benchmarks/udsv/render.ts" + +const result = (overrides: Partial): SweepResult => ({ + data: "a.csv", + mib: 2, + name: "x", + gmean_ms: 10, + mibs: 200, + rss_mib: 10, + rows: 1000, + error: null, + ...overrides, +}) + +test("renderTable sorts fastest first, scales bars per column, and lists errors last", ({ expect }) => { + const table = renderTable([ + result({ name: "slow", mibs: 100, rss_mib: 5 }), + result({ name: "fast", mibs: 200, rss_mib: 10 }), + result({ name: "broken", mibs: null, gmean_ms: null, rss_mib: null, rows: null, error: "Wrong row count!" }), + ]) + + const lines = table.split("\n") + + expect(lines[0]).toBe("**a.csv (2 MiB, 1K rows), in-memory string**") + expect(lines[1]).toBe("") + expect(lines[2]).toMatch(/^\| Name\s+\| Throughput \(MiB\/s\)\s+\| Peak RSS above baseline \(MiB\)\s+\|$/) + expect(lines[3]).toMatch(/^\| -+ \| -+ \| -+ \|$/) + expect(lines[4]).toMatch(/^\| fast\s+\| ░{40} 200\s+\| ░{40} 10\s+\|$/) + expect(lines[5]).toMatch(/^\| slow\s+\| ░{20} 100\s+\| ░{20} 5\s+\|$/) + expect(lines[6]).toContain("broken") + expect(lines[6]).toContain("Wrong row count!") + expect(new Set(lines.slice(2).map((line) => line.length)).size).toBe(1) +}) + +test("renderTable reports sizes past a GiB in GiB", ({ expect }) => { + expect(renderTable([result({ mib: 1454.2, rows: 2 })]).split("\n")[0]).toBe("**a.csv (1.42 GiB), in-memory string**") +}) + +test("renderTables groups by dataset and mode in first-seen order", ({ expect }) => { + const out = renderTables([ + result({ data: "b.csv" }), + result({ data: "a.csv" }), + result({ data: "b.csv", name: "y (stream)" }), + result({ data: "b.csv", name: "z" }), + ]) + + // Each table opens with its bold caption; the blank line after a caption is not a table break. + const tables = out.split(/\n\n(?=\*\*)/) + + expect(tables).toHaveLength(3) + expect(tables[0]).toContain("b.csv") + expect(tables[0]).toContain("in-memory string") + expect(tables[0]).toContain("| z") + expect(tables[1]).toContain("a.csv") + expect(tables[2]).toContain("streamed from file") + expect(tables[2]).toContain("| y (stream)") +}) From 1a471eaaa01bc732b28aee18be55dae39bc5d05d Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:22 +0200 Subject: [PATCH 02/10] Speed up the CSV cell path: windowed streaming scan, batched rows, fewer allocations - AsyncSequence.next() only awaits the opener on the first pull; the per-pull await was an async frame per row on every streamed format (20-40%). - Batched sources: an upstream marked BATCHED_SOURCE yields arrays walked without an await per element. - AsyncSpliterator.nextWindow() hands out every queued record as one contiguous view, and the streaming CSV path runs the cell kernel over each window, falling back per window on invalid UTF-8. The streaming branch awaits the scanner so a fresh process takes it on its first file. - A string source is the decoded text: no second copy of the source, cells reference the caller's string. - The kernel emits CELL_FLAG_HAS_ESCAPE when a quote opens directly after a quote, carried across windows; the cell loop slices a quoted cell once, inside the quotes, and only unescapes on the flag. The row-path walk unquotes as it splits. - Rows are filled into exact-width template copies instead of grown by push. With rows retained the path was scavenger-bound: 23MB allocated per 2MB parse against uDSV's 8.5MB. Measured: 300MB quoted stream 4.2s -> 2.1s, unquoted 3.2s -> 1.9s, 386MB in-memory string 4.2s -> 3.2s at 2237 -> 1321 MiB peak, litmus in-memory 103 -> 209 MiB/s on uDSV's harness. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- AGENTS.md | 10 +- lib/core/AsyncSpliterator.ts | 46 ++++++++ lib/core/CharacterSequence.ts | 9 +- lib/core/wasm_base64.ts | 2 +- lib/core/wasm_module.ts | 12 ++- lib/formats/CSVSpliterator.ts | 105 +++++++++++++++++-- lib/formats/csv-cells.ts | 157 ++++++++++++++++++++++++---- lib/formats/csv-columns.ts | 67 +++++++++--- lib/iterators/AsyncSequence.ts | 109 ++++++++++++++++--- test/core/QuoteHandling.test.ts | 9 ++ test/core/nextWindow.test.ts | 94 +++++++++++++++++ test/core/wasm.test.ts | 78 +++++++++++--- test/formats/CSVSpliterator.test.ts | 112 ++++++++++++++++++-- test/formats/csv-cells.test.ts | 35 ++++++- test/iterators/batched.test.ts | 100 ++++++++++++++++++ wasm/src/lib.rs | 53 ++++++++-- 16 files changed, 902 insertions(+), 96 deletions(-) create mode 100644 test/core/nextWindow.test.ts create mode 100644 test/iterators/batched.test.ts diff --git a/AGENTS.md b/AGENTS.md index b548049..03a9870 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,12 +154,16 @@ Boyer-Moore-Horspool fallback, and ~8–17× for `searchAll`. The `mode` emitters and transformer-binding shared by `CSVSpliterator` and `XLSXSpliterator` live in `lib/formats/row-emitters.ts`, generalized over the cell type (CSV binds `string` with `""` for missing columns; XLSX binds typed cells with `null`). +- **Object and entries rows are built by a compiled literal after 32 rows of the same header.** `createRowEmitters` keeps a `WeakMap` from the bound header array to a builder; the first 32 rows go through an indexed loop, then `compileBuilder` writes `{ "id": f[0](0 < c.length ? c[0] : m), ... }` with the header names as string-literal keys and compiles it with `new Function`. V8 allocates a literal of known shape in one step where `record[key] = value` was a keyed store per cell: on uDSV's 10k × 20 litmus fixture, object mode went 62ms → 10ms (array mode is 10ms) and entries 38ms → 17ms; the loop alone had taken it to 30ms, the `zipSync` iterator and tuple per cell being half of the original. `JSON.stringify` on the key makes any header a valid literal key, including `__proto__`, which stays a setter exactly as the dynamic store was. A `new Function` that throws (a CSP without `unsafe-eval`) marks the header as not compilable and the loop stays. Columns past the header are appended as `column_` after the literal. The `zipSync` path remains only for non-array iterables. `bindTransformers` reads own properties of a transformer record only, since a header named `constructor` or `__proto__` used to bind `Object.prototype`'s. + All high-level classes are abstract static-only (instantiation throws `TypeError`). They expose `from(syncSource)` and `fromAsync(asyncSource)` class methods. `from` returns a **`Sequence`**; `fromAsync` returns an **`AsyncSequence`**. Both are lazy and chainable, and neither engine class is one — `Spliterator.fromSync` returns a `Spliterator` (it keeps `position`, `toDecodedArray`, `Symbol.dispose`), and `Sequence.from(spliterator)` wraps it when the chain is wanted. ### `AsyncSequence` (`lib/iterators/AsyncSequence.ts`) A lazy, chainable async iterator returned by every `fromAsync`. Core methods (`map`, `filter`, `take`, `drop`, `flatMap`, `reduce`, `toArray`, `forEach`, `some`, `every`, `find`) match the [async iterator helpers proposal](https://github.com/tc39/proposal-async-iterator-helpers) in name, arity, and semantics, including the `counter` argument. Extras that are deliberately not spec surface: `chunks(size)`, `parallelMap(fn, opts)`, `parallelFilter(fn, opts)`, `toReadableStream()`, `pipeThrough()`. +- **Batched sources.** An upstream marked with `BATCHED_SOURCE` (`batched(iterable)`) yields arrays, and `next()` walks each array in place without an `await` per element. The streaming CSV cell scanner uses it to hand over a whole engine window of rows at once. Measured on a 300MB quoted CSV: rows yielded singly from an async generator 4.8s, the per-row path 3.2s, batched 2.9s, and 2.5s with the opener fix below. Clean end-to-end against the previous release on the same file: quoted 4.2s → 2.5s, unquoted 3.2s → 1.9s. A batched source that needs a header row consumed must put the remainder of that batch back in front (`reheadBatches`). +- **`next()` only awaits the opener on the first pull.** It read `this.#upstream ?? this.#syncUpstream` before falling back to `await this.#openUpstream()`; awaiting the async opener on every pull was an async frame per item and cost 20–40% on streamed rows (the windowed CSV scan went 2.9s → 2.5s on a 300MB quoted file, `TextSpliterator` over the same file 2.0s → 1.6s). Keep that read in front of the await. - **Fused, not nested.** A chain is an op list plus a source; iteration runs the ops in a plain loop inside one hand-rolled `next()`. One async boundary per item regardless of chain depth — only the op loop grows. Measured on Node 26 over 2M items: bare async generator 10.3M/s, `AsyncSequence` 5.4M/s at three operators and 4.9M/s at six, nested async generators 2.3M/s. Do not "simplify" this into chained `async function*` wrappers; that is a 2.3× regression. Do not extract the pull loop into a separate `async` method either — the extra async frame per item cost 30% when it was tried. - **`flatMap`, `chunks`, `parallelMap`, and `parallelFilter` are fusion barriers** — they need inner-iterator state, so each starts a fresh segment. - **Callbacks are awaited only when thenable**, so synchronous callbacks cost no microtask hop. @@ -184,7 +188,8 @@ Every `fromAsync` opens its source through `openDelimitedRows`, which reads sour - **The win is fixed setup cost, not throughput.** Opening a handle and standing up a read stream is ~100µs, which dominates a small file and vanishes in a large one. Measured end-to-end, min of 200, two independent sweeps: ~1.85× at 635B, ~1.45× at 6.5KB, ~1.4× at 125KiB, and **nothing reliable at 253KiB or above** — the two sweeps disagreed on the sign there. That is why the default is 128 KiB and not larger; raising it trades memory linearly for a difference that no longer measures. - **A raw synchronous parse looks far better than this path can deliver** (~1.6× even at 1GiB). The rest is eaten by the per-row cost of `AsyncSequence`, which both paths pay. Don't re-derive the raw number and conclude the threshold should be raised. -- **The bulk path awaits `CharacterSequence.whenReady()`.** The sync engine normally misses the WASM scanner because it finishes before the module loads; reaching it through an async path is the one place that can be fixed, worth ~26% on a large source. +- **The bulk path awaits `CharacterSequence.whenReady()`.** The sync engine normally misses the WASM scanner because it finishes before the module loads; reaching it through an async path is the one place that can be fixed, worth ~26% on a large source. `CSVSpliterator.fromAsync` awaits it on the streaming branch too, in `openRows`, because the windowed cell scan below checks `hasScanner()` and a fresh process would otherwise take the row path for its first file. +- **Streaming CSV scans cells per engine window.** `AsyncSpliterator.nextWindow()` dequeues every queued range at once and returns the contiguous view from the first range's start to the last range's end, whole records only, about a high-water mark (64 KiB) of rows per `await`. `scanCsvCellsStreaming` (`lib/formats/csv-cells.ts`) decodes each window with the fatal decoder and runs `scanCsvCells` over it; nothing carries across windows because the engine cut the records with quote state in hand. A window that is not UTF-8 takes the per-row path for that window alone. The rows go out as a batched source (see `AsyncSequence`); yielding them singly was slower than the per-row path. `columnScan: "rows"` opts out, and it remains the parity reference. The engine's `skipEmpty`/`drop`/`take` do not apply to windows; the CSV path applies all three itself. - **Unsized sources use an end-of-input test, not a size test.** Pull one chunk; if the stream is already exhausted the whole input is in hand. Otherwise the pulled chunks are put back in front via a re-headed iterable — `test/io/adaptive-source.test.ts` covers 1/7/64/1024-byte chunkings because losing a chunk here would be silent. - **`bulkThreshold: 0` forces streaming.** Use it when a bounded footprint is the point. - Both engines must agree exactly; the parity tests assert the same fixture through both. @@ -308,6 +313,9 @@ The parallel-parsing layers are tested bottom-up so the worker protocol is verif - **`scanRanges` stages `[scanCursor, end)` into WASM memory per call, so `end` must be a window**: the sync `Spliterator` passes `min(source, scanCursor + 64 KiB)`, never the whole source. Handing it the remainder copied up to the full source per 4 KiB fill and made a 1M-row quoted CSV scan 969ms against 85ms plain; windowed it is 62ms. The record open at a window's edge resumes through the carried `pendingSliceStart`/`insideQuotes`, which is the kernel's contract. Before this the sync quote path scanned the whole remainder through `searchMatches`, one match object per delimiter, and above `WASM_MAX_RESULTS` that call falls back to the JS scanner. - **`scan_csv_cells` counts UTF-16 units from the UTF-8 lead byte, which is only exact for valid UTF-8**: the fast path decodes with `{ fatal: true, ignoreBOM: true }` and treats the decoder's `TypeError` as "take the row path". Decoding lossily would silently misalign every cell after the first bad byte. `ignoreBOM` is required too: the row path strips one BOM per row because it decodes per row, so `csv-cells.ts` strips it per row itself. +- **A string source is the decoded text**: `CSVSpliterator.from(string)` passes the string itself to the cell scanner instead of decoding the bytes it encoded from it. The offsets agree because a lone surrogate encodes as U+FFFD, one UTF-16 unit either way, so the surrogate survives where the row path yields U+FFFD. Worth 11% (4.2s → 3.8s) and 770 MiB of peak memory on a 386MB string, the second copy of the source; retained cells then keep the caller's string alive rather than a copy. `columnScan: "rows"` no longer saves memory on a string source for that reason: it decodes per row and its cells are fresh strings, 1.8 GiB against 1.5 GiB retained. +- **`unquoteColumn` only calls `replaceAll` after something says a doubled quote is there**: the row-path walk `splitQuotedString` carries its own `sawDoubledQuote` and unquotes as it splits rather than slicing each quoted cell twice; `unquoteColumn` itself runs one `indexOf('""')`. The cell-scan path never calls it: the kernel's `CELL_FLAG_HAS_ESCAPE` (a quote that opened directly after a quote, carried across windows through `previous_byte` and the state's `cell_flags`) says whether to unescape, and the cell is sliced from the source text once, inside the quotes. The two-pass form plus unconditional `replaceAll` was 17% of a streamed quoted parse and 30% of the quoted walk; the kernel flag was another 5% on the stream. All three must keep agreeing on what a quoted field means. +- **The cell-scan path is scavenger-bound when rows are retained, so its garbage per cell matters more than its instructions**: on uDSV's 1.9MB `litmus_quoted` fixture the parse ran at a steady 9ms when rows were discarded and 19ms when kept, because it allocated ~23MB per parse against ~8.5MB for uDSV, so V8 scavenged once per parse instead of once per three and each scavenge copied the ~12MB result built so far (`--trace-gc-nvp`: `allocated`, `new_space_survived`). Two allocations per cell were the difference: the quoted outer string sliced before the inner one, and rows grown by `push`, which left a 43-slot store behind 20 cells plus two discarded stores per row. Cells are now sliced once and rows are filled into an exact-size `template.slice()` the width of the previous row. Harness-style geometric mean 17.6ms → 8.7ms (103 → ~210 MiB/s), GC 6.9ms → 0.9ms per parse, and the 300MB quoted stream 2.4s → 2.1s. When a change here looks free in a discard-rows loop, measure it with the rows kept. - **Kernel result views alias shared memory**: `CharacterSequence.scanCells` copies each batch out before returning, because a transformer may run a nested parse while the outer row is being assembled. Don't "optimize" the copy away. - **A static `import` of anything under `out/` from a test breaks the second `tsc -b`** (TS5055, the emitted `.d.ts` becomes its own input). Tests that need un-exported internals load them in a `beforeAll` through a dynamic import whose specifier is a variable; see `test/core/wasm.test.ts`. diff --git a/lib/core/AsyncSpliterator.ts b/lib/core/AsyncSpliterator.ts index 521cfde..569e387 100644 --- a/lib/core/AsyncSpliterator.ts +++ b/lib/core/AsyncSpliterator.ts @@ -741,6 +741,52 @@ export class AsyncSpliterator> { + if (this.#done) return this.#finalize() + + if (!this.#indices.size) { + await this.#fill() + } + + // See `next`: the first fill may read into EOF without flushing the tail. + if (this.#lastReadResult?.done && !this.#indices.size && !this.#done) { + await this.#fill() + } + + const first = this.#indices.dequeue() + + if (!first) { + this.#done = true + + return this.#finalize() + } + + const start = first[0] + let end = first[1] + let range: ByteRange | undefined + + while ((range = this.#indices.dequeue())) { + end = range[1] + } + + this.#yieldedByteLength += end - start + + return { value: this.#controller.subarray(start, end), done: false } + } + public toArray(): Promise { return Array.fromAsync(this, (range) => (range as Uint8Array).slice() as unknown as R) } diff --git a/lib/core/CharacterSequence.ts b/lib/core/CharacterSequence.ts index ef0fa1b..3d05be6 100644 --- a/lib/core/CharacterSequence.ts +++ b/lib/core/CharacterSequence.ts @@ -115,7 +115,10 @@ export interface CellScanState { * Absolute UTF-16 start of the open cell. */ cellStartUnits: number - cellHasQuote: boolean + /** + * `CELL_FLAG_HAS_QUOTE | CELL_FLAG_HAS_ESCAPE` of the open cell. + */ + cellFlags: number } export interface CellScanOptions { @@ -470,7 +473,7 @@ export class CharacterSequence extends Uint8Array { state.insideQuotes ? 1 : 0, // Window-relative; negative when the open cell began before this window. state.cellStartUnits - state.units, - state.cellHasQuote ? 1 : 0, + state.cellFlags, previousByte, resultsOffset, maxCells @@ -493,7 +496,7 @@ export class CharacterSequence extends Uint8Array { units: unitBase + block[1]!, insideQuotes: block[2] === 1, cellStartUnits: unitBase + block[3]!, - cellHasQuote: block[4] === 1, + cellFlags: block[4]!, } } diff --git a/lib/core/wasm_base64.ts b/lib/core/wasm_base64.ts index 28e9d73..efdb488 100644 --- a/lib/core/wasm_base64.ts +++ b/lib/core/wasm_base64.ts @@ -19,4 +19,4 @@ * - `scan_csv_cells(...) -> usize (UTF-16 cell boundaries)` */ export const WASM_BASE64 = - "AGFzbQEAAAABOwVgBH9/f38Bf2AGf39/f39/AX9gB39/f39/f38Bf2AMf39/f39/f39/f39/AX9gCX9/f39/f39/fwF/AwcGAAECAAMEBQMBAAEHbQYGbWVtb3J5AgATZmluZF9hbGxfZGVsaW1pdGVycwABEGZpbmRfYWxsX21hdGNoZXMAAg5maW5kX2RlbGltaXRlcgADDnNjYW5fY3N2X2NlbGxzAAQVc2Nhbl9kZWxpbWl0ZWRfcmFuZ2VzAAUKhhcGxAICDX8BeyACLQAAIQsgASADQQFrIg1rIgRBACABIARPGyIJQRBPBEAgC/0PIREgACEKQRAhBANAIAUhDCAEIQUgACAMav0AAAAgEf0j/WQiBgRAA0AgASAGaCIEIAxqIg4gA2pPBEAgBCAKaiEHIAMhBCACIQgDQCAERQRAIA4PCyAEQQFrIQQgCC0AACAHLQAAIAhBAWohCCAHQQFqIQdGDQALCyAGQQFrIAZxIgYNAAsLIApBEGohCiAFQRBqIgQgCU0NAAsLIAUgCUkEQCACQQFqIQEgACAFakEBaiEGA0AgDSEEIAEhByAGIQgCQCAAIAVqLQAAIAtHDQADQCAEBEAgBEEBayEEIActAAAgCC0AACAHQQFqIQcgCEEBaiEIRg0BDAILCyAFDwsgBkEBaiEGIAVBAWoiBSAJSQ0ACwtBfwuTAgIJfwF7AkAgBUUgAUUgA0VyckUEQCABIANJDQEgA0EBRyENA0AgACAJaiEKIAEgCWshCwJ/IA1FBEAgAi0AACIO/Q8hD0EAIQYCQANAIAZBEGoiByALSw0BIAYgCmogByEG/QAAACAP/SP9ZCIHRQ0ACyAHaCAGQRBragwCCwNAIAYgC08NBSAGIApqIAZBAWohBi0AACAORw0ACyAGQQFrDAELIAogCyACIAMQAAsiBkEASA0CIAQgCEEDdGoiDCAGIAlqIgc2AgQgDCAJNgIAIAEgAyAHaiIJIANqTyAIQQFqIgggBUkiB3ENAAsgBw0BCyAIDwsgBCAIQQN0aiIAIAE2AgQgACAJNgIAIAhBAWoLgwYCC38DewJAIAZFIAMgBHJFIAFFcnINACACIANqIQ4gA0EBRiIQIARBAUZxRQRAA0AgASAMayELAn8CQAJAIBBFBEAgAw0BQf////8HDAMLIAAgDGohCiACLQAAIg39DyESQQAhBwJAA0AgB0EQaiIJIAtLDQEgByAKaiAJIQf9AAAAIBL9I/1kIglFDQALIAloIAdBEGtqDAMLA0AgByALSQRAIAcgCmohCSAHQQFqIQcgDSAJLQAARw0BDAMLC0F/DAILIAAgDGogCyACIAMQAAwBCyAHQQFrCyIJAn8CQAJAIARBAUYEQCAAIAxqIQ0gDi0AACIP/Q8hEkEAIQcCQANAIAdBEGoiCiALSw0BIAcgDWogCiEH/QAAACAS/SP9ZCIKRQ0ACyAKaCAHQRBragwECwNAIAcgC0kEQCAHIA1qIQogB0EBaiEHIA8gCi0AAEcNAQwDCwtBfwwDCyAEDQFB/////wcMAgsgB0EBawwBCyAAIAxqIAsgDiAEEAALIgdxQQBIDQIgBSAIQQN0aiILIAcgCUggB0EATnEgCUEASHIiCjYCBCALIAcgCSAKGyAMaiIHNgIAIAhBAWohCCAHIAQgAyAKG2oiDCABTw0CIAYgCEsNAAsMAQsgDi0AACEKIAItAAAhAwJAIAFBEEkNACAK/Q8hEiAD/Q8hE0EQIQdBACECA0AgByEJAkAgACACav0AAAAiFCAT/SP9ZCIEIBQgEv0j/WRyIgdFDQAgBEF/cyEMIAUgCEEDdGohBANAIAQgB2giCyACajYCACAEQQRqIAwgC3ZBAXE2AgAgCEEBaiEIIAdBAWsgB3EiB0UNASAEQQhqIQQgBiAISw0ACwsgCUEQaiIHIAFLDQEgCSECIAYgCEsNAAsLIAEgCU0gBiAITXINAANAAkAgAyAAIAlqLQAAIgJGBH9BAAUgAiAKRw0BQQELIQcgBSAIQQN0aiICIAc2AgQgAiAJNgIAIAhBAWohCAsgCUEBaiIJIAFPDQEgBiAISw0ACwsgCAuSAQIDfwF7QX8hBQJAIANBAWsgAU8NACADQQFGBEAgAi0AACIC/Q8hB0EAIQMDQCABIANBEGoiBEkEQANAIAEgA00NBCAAIANqIANBAWohAy0AACACRw0ACyADQQFrDwsgACADaiAEIQP9AAAAIAf9I/1kIgRFDQALIARoIANBEGtqDwsgACABIAIgAxAAIQULIAULpQcCDn8EeyAIQQBHIQ8gBkEARyEMAn8CQAJAIAFBEEkEQAwBCyAE/Q8hGyAD/Q8hHCAC/Q8hHSAEQQBOIRhBECEIA0AgCCERIB0gACASav0AAAAiGv0j/WQhFCAcIBr9I/1kIhUhCCAa/Qzv7+/v7+/v7+/v7+/v7+/v/Sj9ZCEWIBr9DMDAwMDAwMDAwMDAwMDAwMD9Jf1kQf//A3MhFyAUIBsgGv0j/WQgFXIgCCAYG3IiCARAA0AgFkF/IAhoIgZ0QX9zIg5xaSANaiAOIBdxaWohDiAGIBJqIRMCQAJAAkACQCAUIAZ2QQFxIhlFBEAgFSAGdkEBcQ0BQQEhDyAMQQFzIQwMBAsgDEEBcQ0BIA4hBiAFRQ0CIAkhBiAOIA4gDkEBayIMIBMEfyAAIBNqQQFrLQAABSAGC0ENRxsgByAMShshBgwCCyAOIQYgDEEBcUUNAQtBASEMDAELIAsgEE0EQCAKIA42AgQgCiATNgIADAYLIAogEEEMbGoiDEEcaiAZIA9B/wFxQQF0cjYCACAMQRhqIAY2AgAgDEEUaiAHNgIAIA5BAWohB0EAIQ8gCyAQQQFqIhBLBEBBACEMDAELIAogBzYCBCAKIBNBAWo2AgAMBQsgCEEBayAIcSIIDQALCyAXaSAWaSANamohDSARIRIgEUEQaiIIIAFNDQALCwJAIAEgEUsEQCACQf8BcSEOA0ACfwJAAn8CQAJAIAAgEWoiCC0AACIGIA5GIhJFBEAgBiADQf8BcUYNAUEBIA8gBiAEQf8BcUYgBEEATnEiAhshDyACIAxzDAMLIAxBAXENASANIQIgBUUNAyAJIQwgDSANIA1BAWsiAiARBH8gCEEBay0AAAUgDAtBDUcbIAIgB0gbIQIMAwsgDSECIAxBAXFFDQILQQELIQxBAkEBIAZB7wFLG0EAIAbAQUBOGyANagwBCyALIBBNBEAgCiANNgIEIAogETYCAAwFCyAKIBBBDGxqIghBHGogD0H/AXFBAXQgEnI2AgAgCEEYaiACNgIAIAhBFGogBzYCAEEAIQ9BAkEBIAZB7wFLG0EAIAbAQUBOGyANaiECIA1BAWohByAQQQFqIhAgC08NA0EAIQwgAgshDSARQQFqIhEgAUkNAAsLIAogDTYCBCAKIAE2AgAgDEEBcQwCCyAKIAI2AgQgCiARQQFqNgIAQQAMAQtBAAshCCAKIAc2AgwgCiAINgIIIAogD0H/AXE2AhAgEAvoAwIIfwN7IAEgAyABIANJGyEKIAZBAEchBiAHAn8CQCABIAEgAiABIAJJGyICQRBqIgNPBEAgBf0PIREgBP0PIRIgBUEATiENA0AgAiELIAMhAiAAIAtq/QAAACITIBH9I/1kQQAgDRsiDiATIBL9I/1kIg9yIgMEQANAIANoIQwCQAJAAkACQCAFQQBIDQBBASAMdCIQIA9xDQAgDiAQcQ0BCyAGQQFxRQ0BQQEhBgwCCyAGQQFzIQYMAQsgCyAMaiEGIAggCUsEQCAHIAlBA3RqIgxBEGogBjYCACAMQQxqIAo2AgAgBkEBaiEKIAlBAWohCUEAIQYMAQsgByAKNgIEIAcgBjYCAAwFCyADQQFrIANxIgMNAAsLIAJBEGoiAyABTQ0ACwsgASACSwRAIARB/wFxIQMgBUEATiEEA0ACQCADIAAgAmotAAAiC0cEQCAGIAQgCyAFQf8BcUZxcyEGDAELIAZBAXEEQEEBIQYMAQsgCCAJSwRAIAcgCUEDdGoiBkEQaiACNgIAIAZBDGogCjYCACACQQFqIQogCUEBaiEJQQAhBgwBCyAHIAo2AgQgByACNgIADAMLIAEgAkEBaiICRw0ACwsgByAKNgIEIAcgATYCACAGQQFxDAELQQALNgIIIAkL" + "AGFzbQEAAAABOwVgBH9/f38Bf2AGf39/f39/AX9gB39/f39/f38Bf2AMf39/f39/f39/f39/AX9gCX9/f39/f39/fwF/AwcGAAECAAMEBQMBAAEHbQYGbWVtb3J5AgATZmluZF9hbGxfZGVsaW1pdGVycwABEGZpbmRfYWxsX21hdGNoZXMAAg5maW5kX2RlbGltaXRlcgADDnNjYW5fY3N2X2NlbGxzAAQVc2Nhbl9kZWxpbWl0ZWRfcmFuZ2VzAAUKyhgGxAICDX8BeyACLQAAIQsgASADQQFrIg1rIgRBACABIARPGyIJQRBPBEAgC/0PIREgACEKQRAhBANAIAUhDCAEIQUgACAMav0AAAAgEf0j/WQiBgRAA0AgASAGaCIEIAxqIg4gA2pPBEAgBCAKaiEHIAMhBCACIQgDQCAERQRAIA4PCyAEQQFrIQQgCC0AACAHLQAAIAhBAWohCCAHQQFqIQdGDQALCyAGQQFrIAZxIgYNAAsLIApBEGohCiAFQRBqIgQgCU0NAAsLIAUgCUkEQCACQQFqIQEgACAFakEBaiEGA0AgDSEEIAEhByAGIQgCQCAAIAVqLQAAIAtHDQADQCAEBEAgBEEBayEEIActAAAgCC0AACAHQQFqIQcgCEEBaiEIRg0BDAILCyAFDwsgBkEBaiEGIAVBAWoiBSAJSQ0ACwtBfwuTAgIJfwF7AkAgBUUgAUUgA0VyckUEQCABIANJDQEgA0EBRyENA0AgACAJaiEKIAEgCWshCwJ/IA1FBEAgAi0AACIO/Q8hD0EAIQYCQANAIAZBEGoiByALSw0BIAYgCmogByEG/QAAACAP/SP9ZCIHRQ0ACyAHaCAGQRBragwCCwNAIAYgC08NBSAGIApqIAZBAWohBi0AACAORw0ACyAGQQFrDAELIAogCyACIAMQAAsiBkEASA0CIAQgCEEDdGoiDCAGIAlqIgc2AgQgDCAJNgIAIAEgAyAHaiIJIANqTyAIQQFqIgggBUkiB3ENAAsgBw0BCyAIDwsgBCAIQQN0aiIAIAE2AgQgACAJNgIAIAhBAWoLgwYCC38DewJAIAZFIAMgBHJFIAFFcnINACACIANqIQ4gA0EBRiIQIARBAUZxRQRAA0AgASAMayELAn8CQAJAIBBFBEAgAw0BQf////8HDAMLIAAgDGohCiACLQAAIg39DyESQQAhBwJAA0AgB0EQaiIJIAtLDQEgByAKaiAJIQf9AAAAIBL9I/1kIglFDQALIAloIAdBEGtqDAMLA0AgByALSQRAIAcgCmohCSAHQQFqIQcgDSAJLQAARw0BDAMLC0F/DAILIAAgDGogCyACIAMQAAwBCyAHQQFrCyIJAn8CQAJAIARBAUYEQCAAIAxqIQ0gDi0AACIP/Q8hEkEAIQcCQANAIAdBEGoiCiALSw0BIAcgDWogCiEH/QAAACAS/SP9ZCIKRQ0ACyAKaCAHQRBragwECwNAIAcgC0kEQCAHIA1qIQogB0EBaiEHIA8gCi0AAEcNAQwDCwtBfwwDCyAEDQFB/////wcMAgsgB0EBawwBCyAAIAxqIAsgDiAEEAALIgdxQQBIDQIgBSAIQQN0aiILIAcgCUggB0EATnEgCUEASHIiCjYCBCALIAcgCSAKGyAMaiIHNgIAIAhBAWohCCAHIAQgAyAKG2oiDCABTw0CIAYgCEsNAAsMAQsgDi0AACEKIAItAAAhAwJAIAFBEEkNACAK/Q8hEiAD/Q8hE0EQIQdBACECA0AgByEJAkAgACACav0AAAAiFCAT/SP9ZCIEIBQgEv0j/WRyIgdFDQAgBEF/cyEMIAUgCEEDdGohBANAIAQgB2giCyACajYCACAEQQRqIAwgC3ZBAXE2AgAgCEEBaiEIIAdBAWsgB3EiB0UNASAEQQhqIQQgBiAISw0ACwsgCUEQaiIHIAFLDQEgCSECIAYgCEsNAAsLIAEgCU0gBiAITXINAANAAkAgAyAAIAlqLQAAIgJGBH9BAAUgAiAKRw0BQQELIQcgBSAIQQN0aiICIAc2AgQgAiAJNgIAIAhBAWohCAsgCUEBaiIJIAFPDQEgBiAISw0ACwsgCAuSAQIDfwF7QX8hBQJAIANBAWsgAU8NACADQQFGBEAgAi0AACIC/Q8hB0EAIQMDQCABIANBEGoiBEkEQANAIAEgA00NBCAAIANqIANBAWohAy0AACACRw0ACyADQQFrDwsgACADaiAEIQP9AAAAIAf9I/1kIgRFDQALIARoIANBEGtqDwsgACABIAIgAxAAIQULIAUL6QgCEH8EeyAGQQBHIQwgCEECcUEBdiESIAhBAnZBAXEhDgJAAkAgAUEQSQRADAELIAT9DyEdIAP9DyEeIAL9DyEfIARB/wFxIRogBEEATiEbQRAhCANAIAghESAfIAAgFGr9AAAAIhz9I/1kIRUgHiAc/SP9ZCIWIQggHP0M7+/v7+/v7+/v7+/v7+/v7/0o/WQhFyAc/QzAwMDAwMDAwMDAwMDAwMDA/SX9ZEH//wNzIRggFSAdIBz9I/1kIBZyIAggGxtyIggEQANAIBdBfyAIaCIGdEF/cyIQcWkgD2ogECAYcWlqIRAgBiAUaiETAn8CQAJAAkACQCAVIAZ2QQFxIhlFBEAgFiAGdkEBcQ0BIAxBAXFFBEAgCSEGQQEgDiATBH8gACATakEBay0AAAUgBgsgGkYbIQ4LQQEhEiAMQQFzDAULIAxBAXFFDQEMAwsgECEGIAxBAXENAgwBCyAQIQYgBUUNACAJIQYgECAQIBBBAWsiDCATBH8gACATakEBay0AAAUgBgtBDUcbIAcgDEobIQYLIAsgDU0EQCAKIAc2AgwgCkEANgIIIAogEDYCBCAKIBM2AgAMBwsgCiANQQxsaiIMQRxqIA5B/wFxQQJ0IBlBAnIgGSASG3I2AgAgDEEYaiAGNgIAIAxBFGogBzYCACAQQQFqIQdBACEOIAsgDUEBaiINSwRAQQAhEkEADAILIApBADYCECAKIAc2AgwgCkEANgIIIAogBzYCBCAKIBNBAWo2AgAgDQ8LQQELIQwgCEEBayAIcSIIDQALCyAYaSAXaSAPamohDyARIRQgEUEQaiIIIAFNDQALCwJAIAEgEUsEQCAEQf8BcSEQA0ACfwJAAkACQAJAIAAgEWoiFC0AACIIIAJB/wFxRiITRQRAIAggA0H/AXFGDQEgCCAQRyAEQQBIcg0DIAxBAXFFBEAgCSEGQQEgDiARBH8gFEEBay0AAAUgBgsgEEYbIQ4LQQEhEiAMQQFzIQwMAwsgDEEBcQ0BIA8hBiAFRQ0DIAkhDCAGIAYgBkEBayIGIBEEfyAUQQFrLQAABSAMC0ENRxsgBiAHSBshBgwDCyAPIQYgDEEBcUUNAgtBASEMC0ECQQEgCEHvAUsbQQAgCMBBQE4bIA9qDAELIAsgDU0EQCAKIAc2AgwgCkEANgIIIAogDzYCBCAKIBE2AgAMBQsgCiANQQxsaiIMQRhqIAY2AgAgDEEUaiAHNgIAIAxBHGogDkH/AXFBAnQgE0ECciATIBIbcjYCAEEAIQ5BAkEBIAhB7wFLG0EAIAjAQUBOGyAPaiEGIA9BAWohByANQQFqIg0gC08NA0EAIRJBACEMIAYLIQ8gEUEBaiIRIAFJDQALCyAKIAc2AgwgCiAPNgIEIAogATYCACAKIAxBAXE2AggMAQsgCkEANgIQIAogBzYCDCAKQQA2AgggCiAGNgIEIAogEUEBajYCACANDwsgCiAOQf8BcUECdEECQQAgEhtyNgIQIA0L6AMCCH8DeyABIAMgASADSRshCiAGQQBHIQYgBwJ/AkAgASABIAIgASACSRsiAkEQaiIDTwRAIAX9DyERIAT9DyESIAVBAE4hDQNAIAIhCyADIQIgACALav0AAAAiEyAR/SP9ZEEAIA0bIg4gEyAS/SP9ZCIPciIDBEADQCADaCEMAkACQAJAAkAgBUEASA0AQQEgDHQiECAPcQ0AIA4gEHENAQsgBkEBcUUNAUEBIQYMAgsgBkEBcyEGDAELIAsgDGohBiAIIAlLBEAgByAJQQN0aiIMQRBqIAY2AgAgDEEMaiAKNgIAIAZBAWohCiAJQQFqIQlBACEGDAELIAcgCjYCBCAHIAY2AgAMBQsgA0EBayADcSIDDQALCyACQRBqIgMgAU0NAAsLIAEgAksEQCAEQf8BcSEDIAVBAE4hBANAAkAgAyAAIAJqLQAAIgtHBEAgBiAEIAsgBUH/AXFGcXMhBgwBCyAGQQFxBEBBASEGDAELIAggCUsEQCAHIAlBA3RqIgZBEGogAjYCACAGQQxqIAo2AgAgAkEBaiEKIAlBAWohCUEAIQYMAQsgByAKNgIEIAcgAjYCAAwDCyABIAJBAWoiAkcNAAsLIAcgCjYCBCAHIAE2AgAgBkEBcQwBC0EACzYCCCAJCw==" diff --git a/lib/core/wasm_module.ts b/lib/core/wasm_module.ts index a219c0e..c335081 100644 --- a/lib/core/wasm_module.ts +++ b/lib/core/wasm_module.ts @@ -78,7 +78,7 @@ type WasmScanCsvCells = ( crlf: number, insideQuotes: number, cellStartUnits: number, - cellHasQuote: number, + cellFlags: number, previousByte: number, ro: number, mc: number @@ -128,6 +128,11 @@ export const CELL_RESULT_HEADER = 5 export const CELL_RESULT_STRIDE = 3 export const CELL_FLAG_ROW_END = 1 export const CELL_FLAG_HAS_QUOTE = 2 +/** + * The cell held a quote that opened directly after another quote: a doubled quote inside a quoted field. A cell without + * it needs no search for `""`. + */ +export const CELL_FLAG_HAS_ESCAPE = 4 /** * State returned by the bounded, resumable range scanner. @@ -165,5 +170,8 @@ export interface WasmCellScanResult { * Absolute UTF-16 start of the cell open at `scanCursor`. */ cellStartUnits: number - cellHasQuote: boolean + /** + * `CELL_FLAG_HAS_QUOTE | CELL_FLAG_HAS_ESCAPE` of the open cell. + */ + cellFlags: number } diff --git a/lib/formats/CSVSpliterator.ts b/lib/formats/CSVSpliterator.ts index ad5bedf..453d9a5 100644 --- a/lib/formats/CSVSpliterator.ts +++ b/lib/formats/CSVSpliterator.ts @@ -10,9 +10,15 @@ import { Delimiters, normalizeCharacterInput, } from "#core/CharacterSequence" -import { Spliterator, type SpliteratorInit } from "#core/Spliterator" +import { AsyncSpliterator, Spliterator, type SpliteratorInit } from "#core/Spliterator" import { normalizeColumnNames } from "#formats/casing" -import { cellScanEligibility, decodeForCellScan, scanCsvCells } from "#formats/csv-cells" +import { + cellScanEligibility, + decodeForCellScan, + reheadBatches, + scanCsvCells, + scanCsvCellsStreaming, +} from "#formats/csv-cells" import { splitRowColumns } from "#formats/csv-columns" import { bindTransformers, @@ -27,7 +33,7 @@ import { } from "#formats/row-emitters" import type { AsyncChunkIterator, AsyncDataResource } from "#internal/shared" import { type AdaptiveSourceInit, openDelimitedRows } from "#io/adaptive-source" -import { AsyncSequence } from "#iterators/AsyncSequence" +import { AsyncSequence, BATCHED_SOURCE, type BatchedAsyncIterable } from "#iterators/AsyncSequence" import { Sequence } from "#iterators/Sequence" export type { RowTuple } from "#formats/row-emitters" @@ -151,7 +157,13 @@ function* splitRows(source: CharacterSequenceInput, init: CSVSpliteratorInit, de if (plan && take <= 0 && !header) return // The scanner loads asynchronously; a synchronous caller sees it only if something awaited `whenReady()` first. - const text = plan && CharacterSequence.hasScanner() ? decodeForCellScan(bytes) : null + // + // A string source is already the text the cells are sliced from. Encoding it produced `bytes`, so they are valid + // UTF-8 and the kernel's UTF-16 offsets land on the string the caller passed: a lone surrogate encodes as the three + // bytes of U+FFFD, one unit either way. Decoding here would cost a second copy of the source for every parse, and + // the retained cells would keep that copy alive rather than the caller's own string. + const text = + plan && CharacterSequence.hasScanner() ? (typeof source === "string" ? source : decodeForCellScan(bytes)) : null if (plan && text !== null) { const cellRows = scanCsvCells(bytes, text, { @@ -482,18 +494,95 @@ export abstract class CSVSpliterator { const toColumns = (row: Uint8Array | string[]): string[] => Array.isArray(row) ? row : splitRowColumns(row, columnDelimiter, decoder, enableQuoteHandling, trim) - const openRows = async (): Promise | Iterable> => { + /** + * The streaming counterpart of the bulk branch: the same eligibility, minus the size cap and `position`, which the + * engine has already applied by the time it hands out a window. Returns the rows as the engine gave them when the + * cell scanner does not apply. + */ + const scanWindows = ( + rows: AsyncIterable | Iterable + ): AsyncIterable | BatchedAsyncIterable | Iterable => { + if (!(rows instanceof AsyncSpliterator) || !CharacterSequence.hasScanner()) return rows + + const plan = cellScanEligibility({ + columnScan, + rowDelimiter, + columnDelimiter, + enableQuoteHandling, + position: undefined, + byteLength: 0, + }) + + if (!plan) return rows + + return scanCsvCellsStreaming( + rows, + { + rowDelimiter: plan.rowDelimiter, + columnDelimiter: plan.columnDelimiter, + enableQuoteHandling, + crlf, + trim, + skipEmpty, + }, + (window) => { + const fallbackRows: string[][] = [] + + for (const row of Spliterator.fromSync(window, { + ...rowInit, + position: undefined, + crlf, + enableQuoteHandling, + })) { + fallbackRows.push(toColumns(row)) + } + + return fallbackRows + } + ) + } + + const openRows = async (): Promise< + AsyncIterable | BatchedAsyncIterable | Iterable + > => { // Quote handling applies at both levels: rows must not split on newlines inside quotes, // columns must not split on quoted column delimiters. - const rows = await openDelimitedRows(source, { ...rowInit, crlf, enableQuoteHandling }, bulkParser) + const opened = await openDelimitedRows(source, { ...rowInit, crlf, enableQuoteHandling }, bulkParser) + + // The bulk branch awaits the scanner inside `openDelimitedRows`; the streaming branch has to do it here, or a + // fresh process would take the row path for every file on its first parse. + await CharacterSequence.whenReady() + + const rows = scanWindows(opened) if (header) { // Both engines return `this` from their iterator method. Consuming the header row here advances the cursor the - // row ops will read. Returning `rows` afterwards resumes at row two rather than row one. + // row ops will read. Returning `rows` afterwards resumes at row two rather than row one. The windowed scanner + // hands out batches instead, so its first batch loses its head row and the remainder is put back in front. const iterator = Symbol.asyncIterator in rows ? rows[Symbol.asyncIterator]() : rows[Symbol.iterator]() try { - const result = await iterator.next() + if (BATCHED_SOURCE in rows) { + const batches = iterator as AsyncIterator + let result = await batches.next() + + while (!result.done && !result.value.length) { + result = await batches.next() + } + + if (result.done) return rows + + const [columns, ...remainder] = result.value + const headers = normalizeKeys ? normalizeColumnNames(columns!) : columns! + + transformers = bindTransformers(headers, transformersInput) + + return reheadBatches(remainder, batches) + } + + const result = await ( + iterator as AsyncIterator | Iterator + ).next() if (result.done) return rows diff --git a/lib/formats/csv-cells.ts b/lib/formats/csv-cells.ts index e50f4fe..6feff30 100644 --- a/lib/formats/csv-cells.ts +++ b/lib/formats/csv-cells.ts @@ -4,9 +4,17 @@ * @copyright Sister Software */ +import type { AsyncSpliterator } from "#core/AsyncSpliterator" import { CharacterSequence, type CellScanState } from "#core/CharacterSequence" -import { CELL_FLAG_HAS_QUOTE, CELL_FLAG_ROW_END, CELL_RESULT_STRIDE, WASM_MAX_RESULTS } from "#core/wasm_module" -import { normalizeCell, unquoteColumn } from "#formats/csv-columns" +import { + CELL_FLAG_HAS_ESCAPE, + CELL_FLAG_HAS_QUOTE, + CELL_FLAG_ROW_END, + CELL_RESULT_STRIDE, + WASM_MAX_RESULTS, +} from "#core/wasm_module" +import { normalizeCell } from "#formats/csv-columns" +import { batched, type BatchedAsyncIterable } from "#iterators/AsyncSequence" /** * Bytes staged into the kernel per call. The kernel copies `[cursor, end)` into WASM memory each time, so the window @@ -53,24 +61,55 @@ export function* scanCsvCells(source: Uint8Array, text: string, init: CsvCellSca const options = { rowDelimiter, columnDelimiter, quote: enableQuoteHandling ? 0x22 : -1, crlf, maxCells } const length = source.byteLength - let state: CellScanState = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellHasQuote: false } - let row: string[] = [] + let state: CellScanState = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellFlags: 0 } + // Rows are built into an exact-size copy of a template the width of the previous row, the way a parser that knows + // its column count would preallocate them. Growing a row by `push` left a 43-slot store behind 20 cells and two + // discarded stores per row, and the garbage is what matters here: the scavenger runs once per parse of a 2MB source + // instead of once per three, and each run copies the result built so far. See the note on the generator below. + let template: string[] = [] + let row: string[] = template.slice() + let width = 0 - const cell = (start: number, end: number, hasQuote: boolean): string => { + const cell = (start: number, end: number, flags: number): string => { // The row path decodes each row on its own, which strips one BOM at the row's start. - if (!row.length && start < end && text.charCodeAt(start) === BOM) { + if (width === 0 && start < end && text.charCodeAt(start) === BOM) { start++ } - let value = text.slice(start, end) - - if (hasQuote && enableQuoteHandling) { - value = unquoteColumn(value) + let value: string + + if ( + enableQuoteHandling && + flags & CELL_FLAG_HAS_QUOTE && + end - start >= 2 && + text.charCodeAt(start) === DOUBLE_QUOTE && + text.charCodeAt(end - 1) === DOUBLE_QUOTE + ) { + // Slice the inside directly: the quoted outer string was one more allocation per cell, and the kernel saw every + // quote, so it knows whether there is a doubled one to unescape. Agrees with `unquoteColumn`. + const inner = text.slice(start + 1, end - 1) + + value = flags & CELL_FLAG_HAS_ESCAPE ? inner.replaceAll('""', '"') : inner + } else { + value = text.slice(start, end) } return normalizeCell(value, enableQuoteHandling, trim) } + const finishRow = (): string[] => { + const finished = width === row.length ? row : row.slice(0, width) + + if (width !== template.length) { + template = new Array(width).fill("") + } + + row = template.slice() + width = 0 + + return finished + } + while (state.scanCursor < length) { const end = Math.min(length, state.scanCursor + windowSize) const scan = CharacterSequence.scanCells(source, state, end, options) @@ -89,18 +128,20 @@ export function* scanCsvCells(source: Uint8Array, text: string, init: CsvCellSca const cellEnd = cells[offset + 1]! const flags = cells[offset + 2]! - row.push(cell(start, cellEnd, (flags & CELL_FLAG_HAS_QUOTE) !== 0)) + // Evaluated before the store: `row[width++] = cell(...)` would bump `width` before `cell` reads it. + const value = cell(start, cellEnd, flags) + + row[width++] = value if (flags & CELL_FLAG_ROW_END) { // Empty means the raw row range is empty: one cell, and it spans nothing after CRLF removal. A row's first // cell starts where the row does, so its raw start (before any BOM strip) is the row's start. - const empty = row.length === 1 && cellEnd === start + const empty = width === 1 && cellEnd === start + const finished = finishRow() if (!(empty && skipEmpty)) { - yield row + yield finished } - - row = [] } } @@ -112,12 +153,16 @@ export function* scanCsvCells(source: Uint8Array, text: string, init: CsvCellSca const tailEnd = text.length if (tailStart < tailEnd) { - row.push(cell(tailStart, tailEnd, state.cellHasQuote)) - yield row - } else if (row.length) { + const value = cell(tailStart, tailEnd, state.cellFlags) + + row[width++] = value + yield finishRow() + } else if (width) { // A row whose last cell is empty: `a,` at EOF. - row.push(cell(tailStart, tailEnd, false)) - yield row + const value = cell(tailStart, tailEnd, 0) + + row[width++] = value + yield finishRow() } else if (!skipEmpty) { // An empty source, or one ending on a row delimiter: one empty row, matching String.split. yield [""] @@ -188,3 +233,75 @@ export function decodeForCellScan(bytes: Uint8Array): string | null { throw error } } + +/** + * Drive the cell scanner over a streaming source one engine window at a time, yielding each window's rows as one batch. + * + * Each window the engine hands out spans whole records, so {@linkcode scanCsvCells} runs over it exactly as it would + * over a whole in-memory source, and nothing carries across windows: the engine cut the records with quote state in + * hand, and the scanner's own tail handling closes the window's last row. One decode and one `await` per window + * replaces one of each per row, which is what makes this the streaming counterpart of the in-memory fast path. The rows + * go out as a batch, which {@linkcode batched} tells the sequence to walk without an `await` per row. Yielding them + * singly here was measured slower than the per-row path it replaces. + * + * A window that is not valid UTF-8 goes through `fallback`, the per-row path, for that window alone: the row path + * decodes lossily, so the output is the same as if the whole source had taken it. + * + * `return()` on the iterable closes the engine. Callers that may never start it must close the engine themselves, since + * a never-started generator skips its `finally`. + */ +export function scanCsvCellsStreaming( + engine: AsyncSpliterator, + init: CsvCellScanInit, + fallback: (window: Uint8Array) => string[][] +): BatchedAsyncIterable { + return batched(windows(engine, init, fallback)) +} + +async function* windows( + engine: AsyncSpliterator, + init: CsvCellScanInit, + fallback: (window: Uint8Array) => string[][] +): AsyncGenerator { + try { + while (true) { + const result = await engine.nextWindow() + + if (result.done) return + + const window = result.value + const text = decodeForCellScan(window) + + yield text === null ? fallback(window) : Array.from(scanCsvCells(window, text, init)) + } + } finally { + await engine.return() + } +} + +/** + * A batched iterable that yields `first` and then whatever `rest` has left. Used to put back the remainder of a batch + * whose head row was consumed as the header. `return()` reaches `rest`, so closing still releases the engine. + */ +export function reheadBatches(first: readonly T[], rest: AsyncIterator): BatchedAsyncIterable { + let served = false + + return batched({ + [Symbol.asyncIterator]: () => ({ + next: async () => { + if (!served) { + served = true + + return { value: first, done: false } + } + + return rest.next() + }, + return: async () => { + await rest.return?.() + + return { value: undefined, done: true } + }, + }), + }) +} diff --git a/lib/formats/csv-columns.ts b/lib/formats/csv-columns.ts index ebdb006..b34ff3f 100644 --- a/lib/formats/csv-columns.ts +++ b/lib/formats/csv-columns.ts @@ -26,8 +26,17 @@ function decodeColumn(bytes: Uint8Array, decoder: TextDecoder, enableQuoteHandli * constant they both reference would not have proved that. */ export function unquoteColumn(value: string): string { - if (value.length >= 2 && value.charCodeAt(0) === DOUBLE_QUOTE_CODE && value.endsWith('"')) { - return value.slice(1, -1).replaceAll('""', '"') + if ( + value.length >= 2 && + value.charCodeAt(0) === DOUBLE_QUOTE_CODE && + value.charCodeAt(value.length - 1) === DOUBLE_QUOTE_CODE + ) { + const inner = value.slice(1, -1) + + // Most quoted cells escape nothing, and `replaceAll` costs a scan plus a copy even when it finds nothing. One + // `indexOf` is the cheaper way to learn that. Measured at 17% of a streamed quoted parse before this check. The + // cell-scan path does not come here: the kernel flags a doubled quote, so it slices the inside directly. + return inner.indexOf('""') === -1 ? inner : inner.replaceAll('""', '"') } return value @@ -86,37 +95,73 @@ function delimiterAsString(columnDelimiter: CharacterSequence): string | null { } /** - * Split one decoded row on `delimiter`, honouring double quotes. + * Split one decoded row on `delimiter`, honouring double quotes, and unquote each field in the same pass. * * Callers must establish that `line` contains a quote. {@linkcode splitRowColumns} does so and uses * `String.prototype.split` otherwise. A quote-free row cannot have a quoted field, so it needs no unquote pass. The * hoisted check lets the caller skip both operations. On a 12-column FCC availability file 96% of rows take that path. + * + * Unquoting here rather than over the split result is what keeps a fully quoted file close to the unquoted one: the + * two-pass form sliced each quoted cell and then sliced it again to drop the quotes, and ran `replaceAll` over every + * quoted cell to find that it had nothing to unescape. Measured over 1M rows of a six-column, every-cell-quoted file: + * 344ms two-pass, 243ms fused, against 185ms for `String.prototype.split` on the same rows. The walk notes whether a + * doubled quote was seen inside the cell and only then pays for `replaceAll`. Must agree with {@linkcode unquoteColumn}, + * which the cell-scan path applies to the same fields. */ function splitQuotedString(line: string, delimiter: string): string[] { const columns: string[] = [] + const length = line.length + const delimiterLength = delimiter.length let sliceStart = 0 let index = 0 let insideQuotes = false + let sawDoubledQuote = false - while (index < line.length) { + while (index < length) { if (line.charCodeAt(index) === DOUBLE_QUOTE_CODE) { + if (insideQuotes && line.charCodeAt(index + 1) === DOUBLE_QUOTE_CODE) { + sawDoubledQuote = true + index += 2 + + continue + } + insideQuotes = !insideQuotes index++ } else if (!insideQuotes && line.startsWith(delimiter, index)) { - columns.push(line.slice(sliceStart, index)) - index += delimiter.length + columns.push(unquoteSlice(line, sliceStart, index, sawDoubledQuote)) + sawDoubledQuote = false + index += delimiterLength sliceStart = index } else { index++ } } - columns.push(line.slice(sliceStart)) + columns.push(unquoteSlice(line, sliceStart, length, sawDoubledQuote)) return columns } +/** + * The cell `[start, end)` of `line`, unquoted the way {@linkcode unquoteColumn} would unquote the slice, without first + * materializing the quoted slice. + */ +function unquoteSlice(line: string, start: number, end: number, sawDoubledQuote: boolean): string { + if ( + end - start >= 2 && + line.charCodeAt(start) === DOUBLE_QUOTE_CODE && + line.charCodeAt(end - 1) === DOUBLE_QUOTE_CODE + ) { + const inner = line.slice(start + 1, end - 1) + + return sawDoubledQuote ? inner.replaceAll('""', '"') : inner + } + + return line.slice(start, end) +} + /** * Split one row's bytes into decoded column strings. * @@ -184,13 +229,7 @@ function splitRowColumnsRaw( // the walk and the unquote pass. if (!enableQuoteHandling || line.indexOf('"') === -1) return line.split(delimiter) - const columns = splitQuotedString(line, delimiter) - - for (let i = 0; i < columns.length; i++) { - columns[i] = unquoteColumn(columns[i]!) - } - - return columns + return splitQuotedString(line, delimiter) } // A delimiter that does not round-trip through UTF-8 keeps the byte scan, through the engine itself: a row is a diff --git a/lib/iterators/AsyncSequence.ts b/lib/iterators/AsyncSequence.ts index 337b3c6..bfccefb 100644 --- a/lib/iterators/AsyncSequence.ts +++ b/lib/iterators/AsyncSequence.ts @@ -45,7 +45,49 @@ function isThenable(value: unknown): value is PromiseLike { export type SequenceSource = | AsyncIterable | Iterable - | (() => AsyncIterable | Iterable | PromiseLike | Iterable>) + | BatchedAsyncIterable + | (() => + | AsyncIterable + | Iterable + | BatchedAsyncIterable + | PromiseLike | Iterable | BatchedAsyncIterable>) + +/** + * Marks an async iterable whose items are arrays of elements to be yielded one at a time. + * + * A source that produces its elements in bulk, such as a scanner that splits a whole buffer window into rows, pays one + * `await` per batch instead of one per element when it hands the sequence the whole array. The pull loop in + * {@linkcode AsyncSequence.next} then walks the array synchronously, so a chain over a batched source costs what a chain + * over a synchronous iterator does between batches. Measured on a 300MB quoted CSV streamed through the cell scanner: + * yielding rows one at a time from an async generator was slower than the per-row path it replaced (4.8s against 3.2s), + * because every row still crossed an async boundary; the same rows handed over as batches took 2.9s, and 2.5s once + * `next` stopped awaiting the opener per pull. The per-row async boundary had been a third of the time. + * + * @internal + */ +export const BATCHED_SOURCE: unique symbol = Symbol("spliterator.batchedSource") + +/** + * An async iterable of batches that the sequence flattens one element at a time. + * + * @internal + */ +export interface BatchedAsyncIterable extends AsyncIterable { + readonly [BATCHED_SOURCE]: true +} + +/** + * Mark an async iterable of arrays as a batched source. The sequence yields each array's elements in order. + * + * @internal + */ +export function batched(source: AsyncIterable): BatchedAsyncIterable { + return Object.assign(source, { [BATCHED_SOURCE]: true as const }) +} + +function isBatched(source: object): source is BatchedAsyncIterable { + return BATCHED_SOURCE in source +} function toAsyncIterator(source: AsyncIterable | Iterable): AsyncIterator { if (Symbol.asyncIterator in source) return source[Symbol.asyncIterator]() @@ -323,6 +365,13 @@ export class AsyncSequence implements AsyncIterableIterator, AsyncDisposab #upstream: AsyncIterator | null = null #syncUpstream: Iterator | null = null + + /** + * Set when the upstream is a {@linkcode BatchedAsyncIterable}: its results are arrays that `next` walks in place. + */ + #batched = false + #batch: readonly unknown[] | null = null + #batchIndex = 0 #counters: number[] | null = null #budgets: number[] | null = null #done = false @@ -381,6 +430,7 @@ export class AsyncSequence implements AsyncIterableIterator, AsyncDisposab return this.#syncUpstream } + this.#batched = isBatched(resolved) this.#upstream = toAsyncIterator(resolved) return this.#upstream @@ -765,31 +815,59 @@ export class AsyncSequence implements AsyncIterableIterator, AsyncDisposab if (budgets[index]! <= 0) return this.#finish() } - let upstream: AsyncIterator | Iterator + // Opening is async, but after the first pull the upstream is in hand. Awaiting the opener each time would put an + // async frame on every item; see the file header on what that costs. + let upstream: AsyncIterator | Iterator | null = this.#upstream ?? this.#syncUpstream - try { - upstream = await this.#openUpstream() - } catch (error) { - // A source that cannot open is not retried: the next pull reports done rather than invoking the thunk again. - this.#done = true + if (upstream === null) { + try { + upstream = await this.#openUpstream() + } catch (error) { + // A source that cannot open is not retried: the next pull reports done rather than invoking the thunk again. + this.#done = true - throw error + throw error + } } const isSync = upstream === this.#syncUpstream try { outer: for (;;) { - const pulled = upstream.next() - const result = isSync ? (pulled as IteratorResult) : await pulled + let value: unknown - if (result.done) { - this.#done = true + if (this.#batched) { + // A batched upstream hands over arrays. Walk the current one without touching the event loop and pull + // the next only when it is spent; an empty batch is skipped the same way. + if (this.#batch === null || this.#batchIndex >= this.#batch.length) { + const result = await (upstream as AsyncIterator).next() - return { value: undefined, done: true } - } + if (result.done) { + this.#done = true + this.#batch = null + + return { value: undefined, done: true } + } - let value: unknown = result.value + this.#batch = result.value + this.#batchIndex = 0 + + if (!this.#batch.length) continue outer + } + + value = this.#batch[this.#batchIndex++] + } else { + const pulled = upstream.next() + const result = isSync ? (pulled as IteratorResult) : await pulled + + if (result.done) { + this.#done = true + + return { value: undefined, done: true } + } + + value = result.value + } for (let i = 0; i < length; i++) { const op = ops[i]! @@ -867,6 +945,7 @@ export class AsyncSequence implements AsyncIterableIterator, AsyncDisposab this.#upstream = null this.#syncUpstream = null + this.#batch = null await upstream?.return?.() diff --git a/test/core/QuoteHandling.test.ts b/test/core/QuoteHandling.test.ts index 46ac4d4..6cad7f3 100644 --- a/test/core/QuoteHandling.test.ts +++ b/test/core/QuoteHandling.test.ts @@ -136,6 +136,15 @@ test("CSV: doubled quotes unescape", ({ expect }) => { expect(rows).toEqual([{ h1: 'a"b', h2: "c" }]) }) +test("CSV: the fused quoted walk agrees with unquoteColumn on the awkward shapes", ({ expect }) => { + // Empty quoted cell, doubled quote only, doubled quote after a plain cell, quotes mid-cell (left alone, since the + // cell does not begin with one), a doubled quote in the last cell, and an unterminated quote running to EOF. + const source = encoder.encode('"",""""\nx,"a""b"\na"b"c,d\n"e","f""g"\n"open,d') + const rows = Array.from(CSVSpliterator.from(source, { mode: "array", header: false, trim: false })) + + expect(rows).toEqual([["", '"'], ["x", 'a"b'], ['a"b"c', "d"], ["e", 'f"g'], ['"open,d']]) +}) + test("CSV: empty fields preserved under quote handling", ({ expect }) => { const source = encoder.encode('h1,h2,h3\n"a",,c\n') const rows = Array.from(CSVSpliterator.from(source, { mode: "object", enableQuoteHandling: true })) diff --git a/test/core/nextWindow.test.ts b/test/core/nextWindow.test.ts new file mode 100644 index 0000000..f285e16 --- /dev/null +++ b/test/core/nextWindow.test.ts @@ -0,0 +1,94 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { AsyncSpliterator } from "spliterator" +import { describe, expect, test } from "vitest" + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +async function* chunks(...parts: string[]) { + for (const part of parts) { + yield encoder.encode(part) + } +} + +async function windows( + source: AsyncIterable, + init: ConstructorParameters[1] = {} +) { + const engine = AsyncSpliterator.from(source, { delimiter: "\n", ...init }) + const out: string[] = [] + + for (;;) { + const result = await engine.nextWindow() + + if (result.done) break + + out.push(decoder.decode(result.value)) + } + + return out +} + +describe("AsyncSpliterator.nextWindow", () => { + // Every window boundary sits on a record delimiter, so joining the windows with it reproduces the source. A source + // that ends on a delimiter yields a final empty window: the tail `next` would have handed out as an empty record. + test("windows joined by the delimiter reproduce the source, trailing empty window included", async () => { + const out = await windows(chunks("a,b\nc,d\ne,f\n")) + + expect(out.join("\n")).toBe("a,b\nc,d\ne,f\n") + expect(out.at(-1)).toBe("") + }) + + test("a window never ends in the carriage return of a CRLF; interior ones stay for the consumer", async () => { + const out = await windows(chunks("a\r\nb\r\nc\r\n"), { crlf: true }) + + expect(out.join("\r\n")).toBe("a\r\nb\r\nc\r\n") + expect(out.some((window) => window.endsWith("\r"))).toBe(false) + }) + + test("a tail without a trailing delimiter is the end of the last window", async () => { + const out = await windows(chunks("a\nb\nc")) + + expect(out.join("\n")).toBe("a\nb\nc") + expect(out.at(-1)!.endsWith("c")).toBe(true) + }) + + test("empty records stay in the window even with skipEmpty, for the consumer to drop", async () => { + const out = await windows(chunks("a\n\n\nb\n"), { skipEmpty: true }) + + expect(out.join("\n")).toBe("a\n\n\nb\n") + }) + + test("a quoted region never straddles two windows", async () => { + const quoted = `"x\ny\nz"` + const out = await windows(chunks("a\n", quoted, "\nb\n"), { enableQuoteHandling: true, highWaterMark: 4 }) + + expect(out.join("\n")).toBe(`a\n${quoted}\nb\n`) + expect(out.some((window) => window.includes(quoted))).toBe(true) + }) + + test("many small windows over a chunked source reproduce it exactly", async () => { + const text = Array.from({ length: 500 }, (_, i) => `row ${i},${"v".repeat(i % 7)}`).join("\n") + "\n" + const parts = text.match(/[\s\S]{1,37}/g)! + const out = await windows(chunks(...parts), { highWaterMark: 256 }) + + expect(out.length).toBeGreaterThan(5) + expect(out.join("\n")).toBe(text) + }) + + test("after the last window it reports done, and keeps doing so", async () => { + const engine = AsyncSpliterator.from(chunks("a\nb"), { delimiter: "\n" }) + + while (!(await engine.nextWindow()).done) { + // drain + } + + expect((await engine.nextWindow()).done).toBe(true) + expect((await engine.nextWindow()).done).toBe(true) + }) +}) diff --git a/test/core/wasm.test.ts b/test/core/wasm.test.ts index b880ee8..c92e2a9 100644 --- a/test/core/wasm.test.ts +++ b/test/core/wasm.test.ts @@ -37,13 +37,19 @@ describe("WASM SIMD scanner", () => { let CELL_RESULT_STRIDE: number let CELL_FLAG_ROW_END: number let CELL_FLAG_HAS_QUOTE: number + let CELL_FLAG_HAS_ESCAPE: number beforeAll(async () => { const wasmModulePath = "../../out/lib/core/wasm_module" - ;({ loadWasmModule, CELL_RESULT_HEADER, CELL_RESULT_STRIDE, CELL_FLAG_ROW_END, CELL_FLAG_HAS_QUOTE } = await import( - wasmModulePath - )) + ;({ + loadWasmModule, + CELL_RESULT_HEADER, + CELL_RESULT_STRIDE, + CELL_FLAG_ROW_END, + CELL_FLAG_HAS_QUOTE, + CELL_FLAG_HAS_ESCAPE, + } = await import(wasmModulePath)) }) test("whenReady() resolves true once the SIMD scanner is loaded", async () => { @@ -438,7 +444,7 @@ describe("WASM SIMD scanner", () => { crlf = 1, insideQuotes = 0, cellStartUnits = 0, - cellHasQuote = 0, + cellFlags = 0, previousByte = -1, maxCells = 64, } = {} @@ -465,7 +471,7 @@ describe("WASM SIMD scanner", () => { crlf, insideQuotes, cellStartUnits, - cellHasQuote, + cellFlags, previousByte, resultsOffset, maxCells @@ -485,7 +491,7 @@ describe("WASM SIMD scanner", () => { units: block[1]!, insideQuotes: block[2]!, cellStartUnits: block[3]!, - cellHasQuote: block[4]!, + cellFlags: block[4]!, cells, } } @@ -499,7 +505,7 @@ describe("WASM SIMD scanner", () => { [5, 7, 0], ]) - expect(result).toMatchObject({ cursor: 9, units: 9, insideQuotes: 0, cellStartUnits: 8, cellHasQuote: 0 }) + expect(result).toMatchObject({ cursor: 9, units: 9, insideQuotes: 0, cellStartUnits: 8, cellFlags: 0 }) }) test("a delimiter inside quotes is data, and the cell is flagged", async () => { @@ -602,9 +608,19 @@ describe("WASM SIMD scanner", () => { test("carries an open quoted cell across calls", async () => { const first = await scan(encoder.encode('"ab')) - expect(first).toMatchObject({ cursor: 3, units: 3, insideQuotes: 1, cellStartUnits: 0, cellHasQuote: 1 }) + expect(first).toMatchObject({ + cursor: 3, + units: 3, + insideQuotes: 1, + cellStartUnits: 0, + cellFlags: CELL_FLAG_HAS_QUOTE, + }) - const second = await scan(encoder.encode('c",d\n'), { insideQuotes: 1, cellStartUnits: -3, cellHasQuote: 1 }) + const second = await scan(encoder.encode('c",d\n'), { + insideQuotes: 1, + cellStartUnits: -3, + cellFlags: CELL_FLAG_HAS_QUOTE, + }) expect(second.cells).toEqual([ [-3, 2, CELL_FLAG_HAS_QUOTE], @@ -621,12 +637,41 @@ describe("WASM SIMD scanner", () => { ]) }) + test("a doubled quote marks the cell escaped; an empty quoted cell is only quoted", async () => { + const result = await scan(encoder.encode('"a""b","",c\n')) + + expect(result.cells).toEqual([ + [0, 6, CELL_FLAG_HAS_QUOTE | CELL_FLAG_HAS_ESCAPE], + [7, 9, CELL_FLAG_HAS_QUOTE], + [10, 11, CELL_FLAG_ROW_END], + ]) + }) + + test("a doubled quote split by the window edge is seen through the carried byte", async () => { + // The window before this one ended on the first quote of `""`, so it closed the field and reported no escape. + const result = await scan(encoder.encode('"b",c\n'), { + insideQuotes: 0, + cellStartUnits: -3, + cellFlags: CELL_FLAG_HAS_QUOTE, + previousByte: 0x22, + }) + + expect(result.cells).toEqual([ + [-3, 3, CELL_FLAG_HAS_QUOTE | CELL_FLAG_HAS_ESCAPE], + [4, 5, CELL_FLAG_ROW_END], + ]) + }) + test("a window longer than 16 bytes with matches in every vector agrees with a scalar oracle", async () => { // Quoted items must not embed a real comma: the oracle below splits on every comma in // the raw text without tracking quote state, which is only valid when no quoted item's // span actually contains one (quote-inside-comma masking has its own dedicated test // above). This test's job is vector-boundary agreement across 200 varying-length items. - const text = Array.from({ length: 200 }, (_, i) => (i % 3 === 0 ? `"q${i}x"` : `c${i}é`)).join(",") + "\n" + const text = + Array.from({ length: 200 }, (_, i) => (i % 6 === 0 ? `"q""${i}"` : i % 3 === 0 ? `"q${i}x"` : `c${i}é`)).join( + "," + ) + "\n" + const bytes = encoder.encode(text) const result = await scan(bytes, { maxCells: 4096 }) const expected: Array<[number, number, number]> = [] @@ -636,7 +681,12 @@ describe("WASM SIMD scanner", () => { if (text[i] === ",") { const cell = text.slice(start, i) - expected.push([start, i, cell.includes('"') ? CELL_FLAG_HAS_QUOTE : 0]) + expected.push([ + start, + i, + (cell.includes('"') ? CELL_FLAG_HAS_QUOTE : 0) | (cell.includes('""') ? CELL_FLAG_HAS_ESCAPE : 0), + ]) + start = i + 1 } } @@ -648,7 +698,7 @@ describe("WASM SIMD scanner", () => { describe("CharacterSequence.scanCells", () => { const options = { rowDelimiter: 0x0a, columnDelimiter: 0x2c, quote: 0x22, crlf: true } - const initial = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellHasQuote: false } + const initial = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellFlags: 0 } test("rebases a window's cells and state to absolute offsets", () => { const bytes = encoder.encode("é,a\nbb,c") @@ -662,7 +712,7 @@ describe("WASM SIMD scanner", () => { units: 4, insideQuotes: false, cellStartUnits: 4, - cellHasQuote: false, + cellFlags: 0, }) const second = CharacterSequence.scanCells(bytes, first, bytes.length, options)! @@ -682,7 +732,7 @@ describe("WASM SIMD scanner", () => { units: 4, insideQuotes: true, cellStartUnits: 2, - cellHasQuote: true, + cellFlags: CELL_FLAG_HAS_QUOTE, }) const second = CharacterSequence.scanCells(bytes, first, bytes.length, options)! diff --git a/test/formats/CSVSpliterator.test.ts b/test/formats/CSVSpliterator.test.ts index 7bc4feb..b0c41c6 100644 --- a/test/formats/CSVSpliterator.test.ts +++ b/test/formats/CSVSpliterator.test.ts @@ -513,25 +513,30 @@ describe("columnScan", () => { }) }) - test("the async bulk branch takes the fast path and matches streaming and rows", async ({ expect }) => { + test("the async bulk and streaming branches both take the fast path and match rows", async ({ expect }) => { const spy = vi.spyOn(CharacterSequence, "scanCells") const auto = await CSVSpliterator.fromAsync(fixturePath).toArray() const autoCalls = spy.mock.calls.length spy.mockClear() - const rows = await CSVSpliterator.fromAsync(fixturePath, { columnScan: "rows" }).toArray() const streamed = await CSVSpliterator.fromAsync(fixturePath, { bulkThreshold: 0 } as never).toArray() - const otherCalls = spy.mock.calls.length + const streamedCalls = spy.mock.calls.length + + spy.mockClear() + + const rows = await CSVSpliterator.fromAsync(fixturePath, { columnScan: "rows" }).toArray() + const rowCalls = spy.mock.calls.length spy.mockRestore() expect(autoCalls).toBeGreaterThan(0) - expect(otherCalls).toBe(0) + expect(streamedCalls).toBeGreaterThan(0) + expect(rowCalls).toBe(0) expect(auto).toEqual(rows) expect(auto).toEqual(streamed) }) - test("an unsized single-chunk stream takes the fast path; a multi-chunk stream does not", async ({ expect }) => { + test("an unsized stream takes the fast path whether it arrives in one chunk or many", async ({ expect }) => { const bytes = encoder.encode("name,age\nAda,36\nBob,41\n") const one = async function* () { @@ -561,7 +566,102 @@ describe("columnScan", () => { spy.mockRestore() expect(oneCalls).toBeGreaterThan(0) - expect(manyCalls).toBe(0) + expect(manyCalls).toBeGreaterThan(0) + }) + + describe("streaming cell scan", () => { + // A high-water mark this small makes every engine window a handful of rows, so the fixture crosses many. + const tiny = { bulkThreshold: 0, highWaterMark: 32 } as const + + test("many small windows match the row path on the fixture", async ({ expect }) => { + const spy = vi.spyOn(CharacterSequence, "scanCells") + const windowed = await CSVSpliterator.fromAsync(fixturePath, tiny).toArray() + const calls = spy.mock.calls.length + + spy.mockRestore() + + const rows = await CSVSpliterator.fromAsync(fixturePath, { ...tiny, columnScan: "rows" }).toArray() + + expect(calls).toBeGreaterThan(1) + expect(windowed).toEqual(rows) + expect(windowed.length).toBeGreaterThan(1) + }) + + test("a quoted field longer than a window stays one cell", async ({ expect }) => { + const long = Array.from({ length: 40 }, (_, i) => `line ${i}, with a comma`).join("\n") + const text = `h1,h2\n"${long}",after\nx,y\n` + + const source = async function* () { + yield encoder.encode(text) + } + + const init = { ...tiny, mode: "array", header: false } as const + const windowed = await CSVSpliterator.fromAsync(source(), init).toArray() + const rows = await CSVSpliterator.fromAsync(source(), { ...init, columnScan: "rows" }).toArray() + + expect(windowed).toEqual([ + ["h1", "h2"], + [long, "after"], + ["x", "y"], + ]) + + expect(windowed).toEqual(rows) + }) + + test("a window that is not UTF-8 takes the row path for that window alone", async ({ expect }) => { + const good = encoder.encode("h\n") + const bad = new Uint8Array([0x61, 0x2c, 0xff, 0x0a]) + const more = encoder.encode(Array.from({ length: 20 }, (_, i) => `r${i},ok`).join("\n") + "\n") + + const source = async function* () { + yield good + yield bad + yield more + } + + const init = { ...tiny, mode: "array", header: false } as const + const windowed = await CSVSpliterator.fromAsync(source(), init).toArray() + const rows = await CSVSpliterator.fromAsync(source(), { ...init, columnScan: "rows" }).toArray() + + expect(windowed[1]).toEqual(["a", "\uFFFD"]) + expect(windowed).toEqual(rows) + }) + + test("empty rows kept with skipEmpty false match the row path across windows", async ({ expect }) => { + const text = "h\n\n\na\n\nb\n\n" + + const source = async function* () { + yield encoder.encode(text) + } + + const init = { ...tiny, mode: "array", header: false, skipEmpty: false } as const + + const windowed = await CSVSpliterator.fromAsync(source(), init).toArray() + const rows = await CSVSpliterator.fromAsync(source(), { ...init, columnScan: "rows" }).toArray() + + expect(windowed).toEqual(rows) + expect(windowed).toEqual([["h"], [""], [""], ["a"], [""], ["b"], [""], [""]]) + }) + + test("an early exit closes the streaming source", async ({ expect }) => { + let closed = false + + const source = { + async *[Symbol.asyncIterator]() { + try { + yield encoder.encode("name,age\nAda,36\nBob,41\n") + yield encoder.encode("Cy,52\n") + } finally { + closed = true + } + }, + } + + const first = await CSVSpliterator.fromAsync(source, { bulkThreshold: 0 }).take(1).toArray() + + expect(first).toEqual([{ name: "Ada", age: "36" }]) + expect(closed).toBe(true) + }) }) test("drop and take on the async bulk path keep fromAsync's callback order", async ({ expect }) => { diff --git a/test/formats/csv-cells.test.ts b/test/formats/csv-cells.test.ts index 7699d8e..4d81137 100644 --- a/test/formats/csv-cells.test.ts +++ b/test/formats/csv-cells.test.ts @@ -13,7 +13,7 @@ * Window and batch sizes are kept small so edge cases cross boundaries. */ -import { CharacterSequence, Spliterator } from "spliterator" +import { CharacterSequence, CSVSpliterator, Spliterator } from "spliterator" import { beforeAll, describe, expect, test } from "vitest" const encoder = new TextEncoder() @@ -126,6 +126,9 @@ describe("scanCsvCells parity with the row path", () => { ["doubled quote", 'a,"b""c"\n'], ["quote-only cell", '"",a\n'], ["quote-only line is not empty", '""\n'], + ["ragged widths, wider then narrower then wider", "a,b,c\nd\ne,f\ng,h,i,j\n"], + ["quotes not at the cell edges stay", 'x"y"z,"q"w,"",""""\n'], + ["doubled quote in a short cell and a sliced-length cell", '"a""b","a long cell with a ""quote"" inside"\n'], ["unmatched quote runs to eof", 'a,"b,c\nd,e\n'], ["unmatched quote mid-cell", '5" pipe,b\n'], ["quotes off", 'a,"b,c"\n', { enableQuoteHandling: false }], @@ -187,6 +190,18 @@ describe("scanCsvCells parity with the row path", () => { } }) + test("rows are distinct arrays of exactly their own width", () => { + const rows = fast("a,b,c\nd\ne,f\n") + + expect(rows).toEqual([["a", "b", "c"], ["d"], ["e", "f"]]) + expect(new Set(rows).size).toBe(3) + expect(rows.map((row) => Object.keys(row).length)).toEqual([3, 1, 2]) + + // A yielded row is the caller's: mutating it must not leak into the next one. + rows[0]!.push("tail") + expect(rows[1]).toEqual(["d"]) + }) + test("a row wider than the batch and longer than the window", () => { const cells = Array.from({ length: 50 }, (_, i) => `cell${i}`) const text = cells.join(",") + "\n" + cells.join(",") @@ -206,3 +221,21 @@ describe("scanCsvCells parity with the row path", () => { expect(fast(text, {}, 64 * 1024, 4096)).toEqual(reference(text)) }) }) + +describe("a string source is sliced directly", () => { + test("cells are the source's own code units, a lone surrogate included", async () => { + await CharacterSequence.whenReady() + + // Encoding a lone surrogate yields U+FFFD, and the row path decodes that back. The fast path slices the string the + // caller passed, so the surrogate survives: same UTF-16 length either way, which is what keeps the kernel's + // offsets aligned. + const source = "héllo,wörld\n\uD83D,ok\nx,y\n" + const rows = CSVSpliterator.from(source, { mode: "array", header: false, trim: false }).toArray() + + expect(rows).toEqual([ + ["héllo", "wörld"], + ["\uD83D", "ok"], + ["x", "y"], + ]) + }) +}) diff --git a/test/iterators/batched.test.ts b/test/iterators/batched.test.ts new file mode 100644 index 0000000..b405a40 --- /dev/null +++ b/test/iterators/batched.test.ts @@ -0,0 +1,100 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { AsyncSequence } from "spliterator" +import { beforeAll, describe, expect, test } from "vitest" + +// `batched` is internal, so its type is stripped from the published surface; load it like the other internals. +let batched: (source: AsyncIterable) => AsyncIterable + +beforeAll(async () => { + const modulePath = "../../out/lib/iterators/AsyncSequence" + + ;({ batched } = await import(modulePath)) +}) + +describe("batched sources", () => { + test("elements come out one at a time, in order, across batches", async () => { + const source = batched( + (async function* () { + yield [1, 2] + yield [3] + yield [4, 5, 6] + })() + ) + + expect(await AsyncSequence.from(source).toArray()).toEqual([1, 2, 3, 4, 5, 6]) + }) + + test("empty batches are skipped, including a leading one", async () => { + const source = batched( + (async function* () { + yield [] + yield [1] + yield [] + yield [] + yield [2] + yield [] + })() + ) + + expect(await AsyncSequence.from(source).toArray()).toEqual([1, 2]) + }) + + test("ops see the elements, not the batches", async () => { + const source = batched( + (async function* () { + yield [1, 2, 3] + yield [4, 5, 6] + })() + ) + + const out = await AsyncSequence.from(source as AsyncIterable) + .filter((n) => n % 2 === 0) + .map((n, i) => `${i}:${n}`) + .toArray() + + expect(out).toEqual(["0:2", "1:4", "2:6"]) + }) + + test("an early exit closes the source mid-batch and pulls no further batch", async () => { + let pulled = 0 + let closed = false + + const source = batched( + (async function* () { + try { + pulled++ + yield [1, 2, 3] + + pulled++ + + yield [4, 5, 6] + } finally { + closed = true + } + })() + ) + + expect(await AsyncSequence.from(source).take(2).toArray()).toEqual([1, 2]) + + expect(pulled).toBe(1) + expect(closed).toBe(true) + }) + + test("a thunk may resolve to a batched source", async () => { + const source = () => + Promise.resolve( + batched( + (async function* () { + yield ["a", "b"] + })() + ) + ) + + expect(await AsyncSequence.from(source).toArray()).toEqual(["a", "b"]) + }) +}) diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 86a2f91..5faee70 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -305,12 +305,18 @@ const CELL_RESULT_HEADER: usize = 5; const CELL_RESULT_STRIDE: usize = 3; const CELL_FLAG_ROW_END: i32 = 1; const CELL_FLAG_HAS_QUOTE: i32 = 2; +const CELL_FLAG_HAS_ESCAPE: i32 = 4; /// Resumable single-pass CSV cell scan. Emits `[start, end, flags]` per cell in UTF-16 code /// units relative to the window's unit base, tracking quote state and counting units from the /// UTF-8 lead-byte pattern. See docs/superpowers/specs/2026-09-27-csv-bulk-cell-scan-design.md. /// -/// Results block (i32): [cursor, units, inside_quotes, cell_start_units, cell_has_quote, cells...] +/// `flags` carries `HAS_QUOTE` for any quote byte in the cell and `HAS_ESCAPE` when a quote +/// opened directly after another quote, which is a doubled quote inside a quoted field. The +/// consumer skips its own search for `""` on every cell without `HAS_ESCAPE`. +/// +/// Results block (i32): [cursor, units, inside_quotes, cell_start_units, cell_flags, cells...] +/// where `cell_flags` is the open cell's `HAS_QUOTE | HAS_ESCAPE`, carried across calls. #[no_mangle] pub unsafe extern "C" fn scan_csv_cells( haystack_offset: usize, @@ -321,7 +327,7 @@ pub unsafe extern "C" fn scan_csv_cells( crlf: i32, inside_quotes: i32, cell_start_units: i32, - cell_has_quote: i32, + cell_flags: i32, previous_byte: i32, results_offset: usize, max_cells: usize, @@ -338,7 +344,8 @@ pub unsafe extern "C" fn scan_csv_cells( let mut units: i32 = 0; let mut quoted = inside_quotes != 0; let mut cell_start = cell_start_units; - let mut has_quote = cell_has_quote != 0; + let mut has_quote = cell_flags & CELL_FLAG_HAS_QUOTE != 0; + let mut has_escape = cell_flags & CELL_FLAG_HAS_ESCAPE != 0; let mut count = 0usize; let row_splat = i8x16_splat(row_byte as i8); @@ -370,6 +377,7 @@ pub unsafe extern "C" fn scan_csv_cells( max_cells: usize, cell_start: &mut i32, has_quote: &mut bool, + has_escape: &mut bool, units_at: i32, end_units: i32, row_end: bool, @@ -391,6 +399,10 @@ pub unsafe extern "C" fn scan_csv_cells( flags |= CELL_FLAG_HAS_QUOTE; } + if *has_escape { + flags |= CELL_FLAG_HAS_ESCAPE; + } + *results.add(base) = *cell_start; *results.add(base + 1) = end_units; *results.add(base + 2) = flags; @@ -398,6 +410,7 @@ pub unsafe extern "C" fn scan_csv_cells( // The delimiter is one ASCII byte, one unit. *cell_start = units_at + 1; *has_quote = false; + *has_escape = false; true } @@ -426,6 +439,14 @@ pub unsafe extern "C" fn scan_csv_cells( } } + // Whether the byte before `offset` is the quote byte, reading the carried byte at the window's edge. + #[inline(always)] + unsafe fn quote_before(haystack: *const u8, offset: usize, previous_byte: i32, quote_byte: u8) -> bool { + let before = if offset == 0 { previous_byte } else { *haystack.add(offset - 1) as i32 }; + + before == quote_byte as i32 + } + while cursor + 16 <= haystack_len { let chunk = v128_load(haystack.add(cursor) as *const v128); let row_mask = i8x16_bitmask(i8x16_eq(chunk, row_splat)) as u32; @@ -457,19 +478,24 @@ pub unsafe extern "C" fn scan_csv_cells( units_at }; - if !close_cell(results, &mut count, max_cells, &mut cell_start, &mut has_quote, units_at, end_units, is_row) { - write_cell_scan_state(results, offset, units_at, quoted, cell_start, has_quote); + if !close_cell(results, &mut count, max_cells, &mut cell_start, &mut has_quote, &mut has_escape, units_at, end_units, is_row) { + write_cell_scan_state(results, offset, units_at, quoted, cell_start, has_quote, has_escape); return count; } if count >= max_cells { // The result buffer is now full. The cursor is immediately after the // delimiter that filled it — one ASCII byte, one unit past this position. - write_cell_scan_state(results, offset + 1, units_at + 1, quoted, cell_start, has_quote); + write_cell_scan_state(results, offset + 1, units_at + 1, quoted, cell_start, has_quote, has_escape); return count; } } } else { + // A quote that opens right after a quote is the second half of a doubled quote. + if !quoted && quote_before(haystack, offset, previous_byte, quote_byte) { + has_escape = true; + } + quoted = !quoted; has_quote = true; } @@ -494,14 +520,18 @@ pub unsafe extern "C" fn scan_csv_cells( units }; - if !close_cell(results, &mut count, max_cells, &mut cell_start, &mut has_quote, units, end_units, is_row) { - write_cell_scan_state(results, cursor, units, quoted, cell_start, has_quote); + if !close_cell(results, &mut count, max_cells, &mut cell_start, &mut has_quote, &mut has_escape, units, end_units, is_row) { + write_cell_scan_state(results, cursor, units, quoted, cell_start, has_quote, has_escape); return count; } just_filled = count >= max_cells; } } else if quote_enabled && byte == quote_byte { + if !quoted && quote_before(haystack, cursor, previous_byte, quote_byte) { + has_escape = true; + } + quoted = !quoted; has_quote = true; } @@ -512,12 +542,12 @@ pub unsafe extern "C" fn scan_csv_cells( if just_filled { // The result buffer is now full. The cursor is immediately after the delimiter // that filled it, which was just accounted for above. - write_cell_scan_state(results, cursor, units, quoted, cell_start, has_quote); + write_cell_scan_state(results, cursor, units, quoted, cell_start, has_quote, has_escape); return count; } } - write_cell_scan_state(results, haystack_len, units, quoted, cell_start, has_quote); + write_cell_scan_state(results, haystack_len, units, quoted, cell_start, has_quote, has_escape); count } @@ -529,12 +559,13 @@ unsafe fn write_cell_scan_state( inside_quotes: bool, cell_start_units: i32, cell_has_quote: bool, + cell_has_escape: bool, ) { *results = cursor as i32; *results.add(1) = units; *results.add(2) = if inside_quotes { 1 } else { 0 }; *results.add(3) = cell_start_units; - *results.add(4) = if cell_has_quote { 1 } else { 0 }; + *results.add(4) = (if cell_has_quote { CELL_FLAG_HAS_QUOTE } else { 0 }) | (if cell_has_escape { CELL_FLAG_HAS_ESCAPE } else { 0 }); } /// SIMD double-scan: both patterns are single-byte. From dcae07c0efa516e98d265a33f87d14debd715bbb Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:19:22 +0200 Subject: [PATCH 03/10] Compile object and entries row builders after 32 rows of a header Replace the zipSync iterator with an indexed loop when both arguments are arrays, then compile a per-header literal builder with new Function and cache it on the bound header array. Object mode 62ms -> 10ms on uDSV's 10k x 20 litmus fixture, the same as array mode; entries 38ms -> 17ms. A Function constructor that throws under CSP leaves the loop in place. bindTransformers reads own properties only; a header named __proto__ used to bind Object.prototype as its transformer. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- lib/formats/row-emitters.ts | 172 +++++++++++++++++++++++++++++- test/formats/row-emitters.test.ts | 142 ++++++++++++++++++++++++ 2 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 test/formats/row-emitters.test.ts diff --git a/lib/formats/row-emitters.ts b/lib/formats/row-emitters.ts index 086f6e5..37e05b9 100644 --- a/lib/formats/row-emitters.ts +++ b/lib/formats/row-emitters.ts @@ -87,11 +87,147 @@ export const identity = (value: V): V => value * * `missingValue` fills columns absent from a short row (CSV binds `""`, XLSX binds `null`). */ +/** + * Rows per bound header before the emitter compiles a builder for it. A compile is a `new Function` call, tens of + * microseconds, which a 20-row file would never earn back; a 10k-row file earns it back a hundred times over. + */ +const COMPILE_AFTER_ROWS = 32 + +/** + * Above this many columns the generated source stops being worth it and the loop handles the row. + */ +const COMPILE_MAX_COLUMNS = 2048 + +interface BuilderState { + rows: number + /** + * `null` until compiled, `false` once compilation was tried and refused (a CSP that forbids `new Function`, too many + * columns), so it is not retried per row. + */ + object: ((columns: readonly V[]) => EmittedRecord) | null | false + entries: ((columns: readonly V[]) => RowTuple[]) | null | false +} + +/** + * Generate a builder whose body is an object or array literal with the header's keys written in as constants. + * + * V8 allocates a literal with a known shape in one step and stores each field at a fixed offset, where `record[key] = + * value` over 20 dynamic keys goes through a keyed store per cell. Measured on uDSV's 10k × 20 litmus fixture: 6.7ms + * per parse of dynamic stores against 0.25ms for the literal, with the rows escaping. uDSV's `genToTypedRow` is the + * same trick. Transformers are called through an array captured by the generated function; `JSON.stringify` makes every + * header a valid string-literal key, including `__proto__`, which then sets the prototype exactly as the dynamic store + * did. + */ +function compileBuilder( + mode: "object" | "entries", + headerColumns: readonly RowTransformerEntry[], + missingValue: V +): ((columns: readonly V[]) => never) | false { + if (headerColumns.length > COMPILE_MAX_COLUMNS) return false + + const fields = headerColumns.map(([key], idx) => { + const value = `f[${idx}](${idx} < c.length ? c[${idx}] : m)` + + return mode === "object" ? `${JSON.stringify(key)}: ${value}` : `[${JSON.stringify(key)}, ${value}, ${idx}]` + }) + + const literal = mode === "object" ? `{${fields.join(",\n")}}` : `[${fields.join(",\n")}]` + + try { + // The generated source is built from JSON-encoded header names and integer indices only; no caller string is + // spliced in unquoted. The literal is the point, see the doc comment above. + // oxlint-disable-next-line typescript/no-implied-eval + const make = new Function("f", "m", `return function builtRow(c) { return ${literal} }`) as ( + transformers: RowTransformer[], + missing: V + ) => (columns: readonly V[]) => never + + return make( + headerColumns.map(([, transformer]) => transformer), + missingValue + ) + } catch { + // A Content-Security-Policy without 'unsafe-eval' throws here. The loop below is the same result, slower. + return false + } +} + +/** + * Create the `mode`-keyed emitter table for a cell type. + * + * `missingValue` fills columns absent from a short row (CSV binds `""`, XLSX binds `null`). + * + * Both arguments are arrays on every path in this package, and an array row against a bound header takes the fast paths + * below: an indexed loop, and after {@linkcode COMPILE_AFTER_ROWS} rows for the same header a compiled literal (see + * {@linkcode compileBuilder}). An iterator step and a tuple per cell through `zipSync` was 50% of object mode and the + * dynamic keyed stores most of the rest; the generator path remains for callers that hand in other iterables. + */ export function createRowEmitters(missingValue: V): Record | null> { + // One shared empty header so a header-less row still accumulates toward a compiled builder. + const NO_HEADER: RowTransformerEntry[] = [] + const builders = new WeakMap[], BuilderState>() + + const stateFor = (headerColumns: readonly RowTransformerEntry[]): BuilderState => { + let state = builders.get(headerColumns) + + if (!state) { + state = { rows: 0, object: null, entries: null } + + builders.set(headerColumns, state) + } + + state.rows++ + + return state + } + + // Columns past the header have no name: `column_`, as the loop and generator paths name them. + const appendUnnamed = (record: EmittedRecord, columns: readonly V[], from: number) => { + for (let idx = from; idx < columns.length; idx++) { + record[`column_${idx}`] = columns[idx] + } + + return record + } + return { array: null, - entries(columns: Iterable, headerColumns: Iterable> = []): RowTuple[] { + entries(columns: Iterable, headerColumns: Iterable> = NO_HEADER): RowTuple[] { + if (Array.isArray(columns) && Array.isArray(headerColumns)) { + const state = stateFor(headerColumns) + + if (state.entries === null && state.rows > COMPILE_AFTER_ROWS) { + state.entries = compileBuilder("entries", headerColumns, missingValue) + } + + if (state.entries) { + const tuples = state.entries(columns) + + for (let idx = headerColumns.length; idx < columns.length; idx++) { + tuples.push([`column_${idx}`, columns[idx], idx]) + } + + return tuples + } + + const length = Math.max(columns.length, headerColumns.length) + const tuples = new Array>(length) + + for (let idx = 0; idx < length; idx++) { + const transformer = headerColumns[idx] as RowTransformerEntry | undefined + const value = idx < columns.length ? (columns[idx] as V) : missingValue + + tuples[idx] = [ + transformer ? transformer[0] : `column_${idx}`, + transformer ? transformer[1](value) : value, + idx, + ] + } + + return tuples + } + return Array.from(zipSync(headerColumns, columns), ([transformer, value], idx) => { const key = transformer?.[0] const transform = transformer?.[1] ?? identity @@ -99,7 +235,33 @@ export function createRowEmitters(missingValue: V): Record, headerColumns: Iterable> = []): EmittedRecord { + object(columns: Iterable, headerColumns: Iterable> = NO_HEADER): EmittedRecord { + if (Array.isArray(columns) && Array.isArray(headerColumns)) { + const state = stateFor(headerColumns) + + if (state.object === null && state.rows > COMPILE_AFTER_ROWS) { + state.object = compileBuilder("object", headerColumns, missingValue) + } + + if (state.object) { + const record = state.object(columns) + + return columns.length > headerColumns.length ? appendUnnamed(record, columns, headerColumns.length) : record + } + + const record: EmittedRecord = {} + const length = Math.max(columns.length, headerColumns.length) + + for (let idx = 0; idx < length; idx++) { + const transformer = headerColumns[idx] as RowTransformerEntry | undefined + const value = idx < columns.length ? (columns[idx] as V) : missingValue + + record[transformer ? transformer[0] : `column_${idx}`] = transformer ? transformer[1](value) : value + } + + return record + } + const record: EmittedRecord = {} for (const [transformer, value, idx] of zipSync(headerColumns, columns)) { @@ -128,5 +290,9 @@ export function bindTransformers( ? Object.fromEntries(transformersInput as Iterable>) : (transformersInput as RowTransformerRecord) - return headers.map((columnName) => [columnName, byName[columnName] ?? identity]) + // Own properties only: a header named `constructor` or `__proto__` would otherwise pick up Object.prototype's. + return headers.map((columnName) => [ + columnName, + Object.hasOwn(byName, columnName) ? (byName[columnName] ?? identity) : identity, + ]) } diff --git a/test/formats/row-emitters.test.ts b/test/formats/row-emitters.test.ts new file mode 100644 index 0000000..5e13ad0 --- /dev/null +++ b/test/formats/row-emitters.test.ts @@ -0,0 +1,142 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { CSVSpliterator } from "spliterator" +import { describe, expect, test } from "vitest" + +/** + * The object and entries emitters switch from an indexed loop to a compiled literal after 32 rows of the same header, + * so every expectation here is checked on a row before the switch and a row after it. + */ +const EARLY = 3 +const LATE = 81 + +function csv(header: string, rowOf: (i: number) => string, rows = 100): string { + return [header, ...Array.from({ length: rows }, (_, i) => rowOf(i))].join("\n") + "\n" +} + +describe("row emitters", () => { + test("object rows match the header on both sides of the compile threshold", () => { + const rows = CSVSpliterator.from( + csv("id,name,score", (i) => `${i},n${i},${i * 2}`), + { + mode: "object", + normalizeKeys: false, + trim: false, + } + ).toArray() + + expect(rows).toHaveLength(100) + + for (const i of [EARLY, LATE]) { + expect(rows[i]).toEqual({ id: `${i}`, name: `n${i}`, score: `${i * 2}` }) + expect(Object.keys(rows[i]!)).toEqual(["id", "name", "score"]) + } + }) + + test("short rows fill with the missing value and long rows append unnamed columns", () => { + const rows = CSVSpliterator.from( + csv("a,b,c", (i) => (i % 2 ? `${i},x` : `${i},x,y,extra${i},more`)), + { mode: "object", normalizeKeys: false, trim: false } + ).toArray() + + // Odd rows are short, even rows are long. + for (const i of [EARLY, LATE]) { + expect(rows[i]).toEqual({ a: `${i}`, b: "x", c: "" }) + } + + for (const i of [EARLY - 1, LATE - 1]) { + expect(rows[i]).toEqual({ a: `${i}`, b: "x", c: "y", column_3: `extra${i}`, column_4: "more" }) + } + }) + + test("transformers run through the compiled builder too", () => { + const rows = CSVSpliterator.from( + csv("id,score", (i) => `${i},${i}.5`), + { + mode: "object", + normalizeKeys: false, + transformers: { score: Number }, + } + ).toArray() + + expect(rows[EARLY]).toEqual({ id: `${EARLY}`, score: EARLY + 0.5 }) + expect(rows[LATE]).toEqual({ id: `${LATE}`, score: LATE + 0.5 }) + }) + + test("awkward header names are plain own properties, except __proto__ which stays a setter", () => { + const header = `"it's",a b,"quo""te",__proto__,constructor,"\\n"` + + const rows = CSVSpliterator.from( + csv(header, (i) => `${i},${i},${i},${i},${i},${i}`), + { + mode: "object", + normalizeKeys: false, + trim: false, + } + ).toArray() + + for (const i of [EARLY, LATE]) { + const row = rows[i]! + + expect(Object.keys(row)).toEqual(["it's", "a b", 'quo"te', "constructor", "\\n"]) + expect(row['quo"te']).toBe(`${i}`) + expect(row.constructor).toBe(`${i}`) + // A string assigned to __proto__ is ignored by the setter on both paths. + expect(Object.getPrototypeOf(row)).toBe(Object.prototype) + } + }) + + test("duplicate header names: the last column wins on both paths", () => { + const rows = CSVSpliterator.from( + csv("k,k", (i) => `first${i},second${i}`), + { + mode: "object", + normalizeKeys: false, + trim: false, + } + ).toArray() + + expect(rows[EARLY]).toEqual({ k: `second${EARLY}` }) + expect(rows[LATE]).toEqual({ k: `second${LATE}` }) + }) + + test("entries rows carry key, value and index on both sides of the threshold", () => { + const rows = CSVSpliterator.from( + csv("a,b", (i) => (i % 2 ? `${i}` : `${i},x,y`)), + { + mode: "entries", + normalizeKeys: false, + trim: false, + } + ).toArray() + + for (const i of [EARLY, LATE]) { + expect(rows[i]).toEqual([ + ["a", `${i}`, 0], + ["b", "", 1], + ]) + } + + for (const i of [EARLY - 1, LATE - 1]) { + expect(rows[i]).toEqual([ + ["a", `${i}`, 0], + ["b", "x", 1], + ["column_2", "y", 2], + ]) + } + }) + + test("object mode without a header names every column", () => { + const rows = CSVSpliterator.from( + csv("x,y", (i) => `${i},${i}`), + { mode: "object", header: false } + ).toArray() + + expect(rows[0]).toEqual({ column_0: "x", column_1: "y" }) + expect(rows[LATE]).toEqual({ column_0: `${LATE - 1}`, column_1: `${LATE - 1}` }) + }) +}) From 6df01d980f77ad3c256e23570428e12b628dcbbc Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:24:26 +0200 Subject: [PATCH 04/10] Step the cell scanner by kernel batch instead of yielding per row CellRowScanner.nextBatch(out) pushes one kernel batch of rows into the caller's array. The in-memory parser yields a fresh array per batch and walks it in its own generator, so a row crosses one generator rather than two; the streaming path collects a whole engine window into one array with no flattening copy. litmus in-memory 9.7ms -> 8.3ms, 300MB quoted stream 2.1s -> 1.9s. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- AGENTS.md | 4 +- lib/formats/CSVSpliterator.ts | 35 +++-- lib/formats/csv-cells.ts | 255 ++++++++++++++++++++++++---------- 3 files changed, 200 insertions(+), 94 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 03a9870..edbf6cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,7 +132,7 @@ Boyer-Moore-Horspool fallback, and ~8–17× for `searchAll`. `trim: false` for byte parity. **`columnScan: "auto"`** (default) parses a wholly in-memory source by decoding it once with a `fatal` decoder and slicing cells at boundaries the `scan_csv_cells` kernel emits in UTF-16 units; `"rows"` is the per-row path and the parity - reference. `lib/formats/csv-cells.ts` drives the kernel in 64 KiB windows; `cellScanEligibility` + reference. `lib/formats/csv-cells.ts` drives the kernel in 64 KiB windows through `CellRowScanner`, a stepping object whose `nextBatch(out)` pushes one kernel batch of rows (≤4096 cells) into the caller's array: the in-memory parser yields a fresh array per batch (`scanCsvCellBatches`) and walks it in its own generator, the streaming path collects a window into one array, and `scanCsvCells` flattens for tests. It was a generator per row under the format's generator; batching and dropping the flattening copy took the litmus in-memory parse 9.7ms → 8.3ms and the 300MB quoted stream 2.1s → 1.9s; `cellScanEligibility` lists the gates (single ASCII delimiters that differ, no CR delimiter, no `position`, under the string length cap, scanner loaded, valid UTF-8). The async path reaches it through the `bulkParser` argument of `openDelimitedRows`, so only sources the adaptive source reads whole @@ -189,7 +189,7 @@ Every `fromAsync` opens its source through `openDelimitedRows`, which reads sour - **The win is fixed setup cost, not throughput.** Opening a handle and standing up a read stream is ~100µs, which dominates a small file and vanishes in a large one. Measured end-to-end, min of 200, two independent sweeps: ~1.85× at 635B, ~1.45× at 6.5KB, ~1.4× at 125KiB, and **nothing reliable at 253KiB or above** — the two sweeps disagreed on the sign there. That is why the default is 128 KiB and not larger; raising it trades memory linearly for a difference that no longer measures. - **A raw synchronous parse looks far better than this path can deliver** (~1.6× even at 1GiB). The rest is eaten by the per-row cost of `AsyncSequence`, which both paths pay. Don't re-derive the raw number and conclude the threshold should be raised. - **The bulk path awaits `CharacterSequence.whenReady()`.** The sync engine normally misses the WASM scanner because it finishes before the module loads; reaching it through an async path is the one place that can be fixed, worth ~26% on a large source. `CSVSpliterator.fromAsync` awaits it on the streaming branch too, in `openRows`, because the windowed cell scan below checks `hasScanner()` and a fresh process would otherwise take the row path for its first file. -- **Streaming CSV scans cells per engine window.** `AsyncSpliterator.nextWindow()` dequeues every queued range at once and returns the contiguous view from the first range's start to the last range's end, whole records only, about a high-water mark (64 KiB) of rows per `await`. `scanCsvCellsStreaming` (`lib/formats/csv-cells.ts`) decodes each window with the fatal decoder and runs `scanCsvCells` over it; nothing carries across windows because the engine cut the records with quote state in hand. A window that is not UTF-8 takes the per-row path for that window alone. The rows go out as a batched source (see `AsyncSequence`); yielding them singly was slower than the per-row path. `columnScan: "rows"` opts out, and it remains the parity reference. The engine's `skipEmpty`/`drop`/`take` do not apply to windows; the CSV path applies all three itself. +- **Streaming CSV scans cells per engine window.** `AsyncSpliterator.nextWindow()` dequeues every queued range at once and returns the contiguous view from the first range's start to the last range's end, whole records only, about a high-water mark (64 KiB) of rows per `await`. `scanCsvCellsStreaming` (`lib/formats/csv-cells.ts`) decodes each window with the fatal decoder and runs the cell scanner over it (`collectCsvCells`, every row of the window pushed into one array); nothing carries across windows because the engine cut the records with quote state in hand. A window that is not UTF-8 takes the per-row path for that window alone. The rows go out as a batched source (see `AsyncSequence`); yielding them singly was slower than the per-row path. `columnScan: "rows"` opts out, and it remains the parity reference. The engine's `skipEmpty`/`drop`/`take` do not apply to windows; the CSV path applies all three itself. - **Unsized sources use an end-of-input test, not a size test.** Pull one chunk; if the stream is already exhausted the whole input is in hand. Otherwise the pulled chunks are put back in front via a re-headed iterable — `test/io/adaptive-source.test.ts` covers 1/7/64/1024-byte chunkings because losing a chunk here would be silent. - **`bulkThreshold: 0` forces streaming.** Use it when a bounded footprint is the point. - Both engines must agree exactly; the parity tests assert the same fixture through both. diff --git a/lib/formats/CSVSpliterator.ts b/lib/formats/CSVSpliterator.ts index 453d9a5..f4d31c7 100644 --- a/lib/formats/CSVSpliterator.ts +++ b/lib/formats/CSVSpliterator.ts @@ -16,6 +16,7 @@ import { cellScanEligibility, decodeForCellScan, reheadBatches, + scanCsvCellBatches, scanCsvCells, scanCsvCellsStreaming, } from "#formats/csv-cells" @@ -166,7 +167,7 @@ function* splitRows(source: CharacterSequenceInput, init: CSVSpliteratorInit, de plan && CharacterSequence.hasScanner() ? (typeof source === "string" ? source : decodeForCellScan(bytes)) : null if (plan && text !== null) { - const cellRows = scanCsvCells(bytes, text, { + const batches = scanCsvCellBatches(bytes, text, { rowDelimiter: plan.rowDelimiter, columnDelimiter: plan.columnDelimiter, enableQuoteHandling, @@ -175,28 +176,32 @@ function* splitRows(source: CharacterSequenceInput, init: CSVSpliteratorInit, de skipEmpty: rowInit.skipEmpty ?? true, }) - if (header) { - const result = cellRows.next() + let headerPending = header - if (result.done) return + // One generator layer per row, not two: the batches are walked here rather than through a flattening generator. + for (const batch of batches) { + for (const columns of batch) { + if (headerPending) { + const headers = normalizeKeys ? normalizeColumnNames(columns) : columns - const headers = normalizeKeys ? normalizeColumnNames(result.value) : result.value + transformers = bindTransformers(headers, transformersInput) + headerPending = false - transformers = bindTransformers(headers, transformersInput) - } + continue + } - for (const columns of cellRows) { - if (yieldCount < drop) { - yieldCount++ + if (yieldCount < drop) { + yieldCount++ - continue - } + continue + } - if (yieldCount >= yieldLimit) break + if (yieldCount >= yieldLimit) return - yield emitter ? emitter(columns, transformers) : columns + yield emitter ? emitter(columns, transformers) : columns - yieldCount++ + yieldCount++ + } } return diff --git a/lib/formats/csv-cells.ts b/lib/formats/csv-cells.ts index 6feff30..5e68000 100644 --- a/lib/formats/csv-cells.ts +++ b/lib/formats/csv-cells.ts @@ -5,7 +5,7 @@ */ import type { AsyncSpliterator } from "#core/AsyncSpliterator" -import { CharacterSequence, type CellScanState } from "#core/CharacterSequence" +import { CharacterSequence, type CellScanOptions, type CellScanState } from "#core/CharacterSequence" import { CELL_FLAG_HAS_ESCAPE, CELL_FLAG_HAS_QUOTE, @@ -42,44 +42,130 @@ export interface CsvCellScanInit { } /** - * Yield every row of a wholly in-memory CSV as `string[]`, slicing cells out of `text` at the boundaries the kernel - * emits. `text` must be `source` decoded with `{ fatal: true, ignoreBOM: true }`: `fatal` is what makes the kernel's - * UTF-16 unit count exact, and `ignoreBOM` keeps every U+FEFF in the string so the offsets line up; the one BOM the row - * path strips at the start of each row is stripped here per row. + * Every row of a wholly in-memory CSV as `string[]`, produced one kernel batch at a time (up to `maxCells` cells, about + * 200 rows of 20 columns), slicing cells out of `text` at the boundaries the kernel emits. `text` must be `source` + * decoded with `{ fatal: true, ignoreBOM: true }`: `fatal` is what makes the kernel's UTF-16 unit count exact, and + * `ignoreBOM` keeps every U+FEFF in the string so the offsets line up; the one BOM the row path strips at the start of + * each row is stripped here per row. * * Row emptiness follows `Spliterator`: a row is empty when its byte range after CRLF removal is empty, before any BOM * removal, unquoting or trimming. `""`, a BOM-only row and `,,` are not empty. * - * The caller has established eligibility (single ASCII delimiters, scanner loaded); this generator yields nothing until - * first pulled and throws only on a broken scanner contract. + * The caller has established eligibility (single ASCII delimiters, scanner loaded). Nothing is scanned until the first + * `nextBatch`, which throws only on a broken scanner contract. + * + * A stepping object rather than a generator because the rows are pushed straight into the caller's array: the in-memory + * parser hands over a fresh array per batch and yields it ({@linkcode scanCsvCellBatches}), the streaming path hands + * over one array per engine window ({@linkcode collectCsvCells}) and never copies. A generator resumption per row had + * been 13% of a streamed parse, and a flattening copy of each batch 9%. */ -export function* scanCsvCells(source: Uint8Array, text: string, init: CsvCellScanInit): Generator { - const { rowDelimiter, columnDelimiter, enableQuoteHandling, crlf, trim, skipEmpty } = init - const windowSize = init.windowSize ?? CELL_SCAN_WINDOW - // A zero batch would return without advancing and loop forever; it is a test-only knob, so clamp it. - const maxCells = Math.max(1, init.maxCells ?? WASM_MAX_RESULTS) - const options = { rowDelimiter, columnDelimiter, quote: enableQuoteHandling ? 0x22 : -1, crlf, maxCells } - const length = source.byteLength - - let state: CellScanState = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellFlags: 0 } +export class CellRowScanner { + readonly #source: Uint8Array + readonly #text: string + readonly #options: CellScanOptions + readonly #windowSize: number + readonly #enableQuoteHandling: boolean + readonly #trim: boolean + readonly #skipEmpty: boolean + + #state: CellScanState = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellFlags: 0 } + // Rows are built into an exact-size copy of a template the width of the previous row, the way a parser that knows // its column count would preallocate them. Growing a row by `push` left a 43-slot store behind 20 cells and two - // discarded stores per row, and the garbage is what matters here: the scavenger runs once per parse of a 2MB source - // instead of once per three, and each run copies the result built so far. See the note on the generator below. - let template: string[] = [] - let row: string[] = template.slice() - let width = 0 + // discarded stores per row, and the garbage is what matters here: the scavenger ran once per parse of a 2MB source + // instead of once per three, and each run copied the result built so far. + #template: string[] = [] + #row: string[] = [] + #width = 0 + #done = false + + constructor(source: Uint8Array, text: string, init: CsvCellScanInit) { + this.#source = source + this.#text = text + this.#windowSize = init.windowSize ?? CELL_SCAN_WINDOW + this.#enableQuoteHandling = init.enableQuoteHandling + this.#trim = init.trim + this.#skipEmpty = init.skipEmpty + + this.#options = { + rowDelimiter: init.rowDelimiter, + columnDelimiter: init.columnDelimiter, + quote: init.enableQuoteHandling ? DOUBLE_QUOTE : -1, + crlf: init.crlf, + // A zero batch would return without advancing and loop forever; it is a test-only knob, so clamp it. + maxCells: Math.max(1, init.maxCells ?? WASM_MAX_RESULTS), + } + } + + /** + * Push the rows of the next kernel batch onto `out`. Returns `false` once the source is exhausted, after the tail row + * has been pushed; a `true` return may have pushed nothing when a batch closed no row. + */ + public nextBatch(out: string[][]): boolean { + if (this.#done) return false + + const source = this.#source + const length = source.byteLength + const state = this.#state + + if (state.scanCursor >= length) { + this.#finish(out) + + return false + } + + const end = Math.min(length, state.scanCursor + this.#windowSize) + const scan = CharacterSequence.scanCells(source, state, end, this.#options) + + if (!scan) { + // The scanner is loaded (the caller checked) and the window is not empty, so this cannot happen; treat it + // as a contract violation rather than looping. + throw new Error("CellRowScanner: the cell scanner returned no batch for a non-empty window") + } + + const cells = scan.cells + const skipEmpty = this.#skipEmpty + + for (let i = 0; i < scan.count; i++) { + const offset = i * CELL_RESULT_STRIDE + const start = cells[offset]! + const cellEnd = cells[offset + 1]! + const flags = cells[offset + 2]! + + // Evaluated before the store: `row[width++] = cell(...)` would bump `width` before `cell` reads it. + const value = this.#cell(start, cellEnd, flags) + + this.#row[this.#width++] = value + + if (flags & CELL_FLAG_ROW_END) { + // Empty means the raw row range is empty: one cell, and it spans nothing after CRLF removal. A row's first + // cell starts where the row does, so its raw start (before any BOM strip) is the row's start. + const empty = this.#width === 1 && cellEnd === start + const finished = this.#finishRow() + + if (!(empty && skipEmpty)) { + out.push(finished) + } + } + } + + this.#state = scan + + return true + } + + #cell(start: number, end: number, flags: number): string { + const text = this.#text - const cell = (start: number, end: number, flags: number): string => { // The row path decodes each row on its own, which strips one BOM at the row's start. - if (width === 0 && start < end && text.charCodeAt(start) === BOM) { + if (this.#width === 0 && start < end && text.charCodeAt(start) === BOM) { start++ } let value: string if ( - enableQuoteHandling && + this.#enableQuoteHandling && flags & CELL_FLAG_HAS_QUOTE && end - start >= 2 && text.charCodeAt(start) === DOUBLE_QUOTE && @@ -94,79 +180,94 @@ export function* scanCsvCells(source: Uint8Array, text: string, init: CsvCellSca value = text.slice(start, end) } - return normalizeCell(value, enableQuoteHandling, trim) + return normalizeCell(value, this.#enableQuoteHandling, this.#trim) } - const finishRow = (): string[] => { + #finishRow(): string[] { + const width = this.#width + const row = this.#row const finished = width === row.length ? row : row.slice(0, width) - if (width !== template.length) { - template = new Array(width).fill("") + if (width !== this.#template.length) { + this.#template = new Array(width).fill("") } - row = template.slice() - width = 0 + this.#row = this.#template.slice() + this.#width = 0 return finished } - while (state.scanCursor < length) { - const end = Math.min(length, state.scanCursor + windowSize) - const scan = CharacterSequence.scanCells(source, state, end, options) - - if (!scan) { - // The scanner is loaded (the caller checked) and the window is not empty, so this cannot happen; treat it - // as a contract violation rather than looping. - throw new Error("scanCsvCells: the cell scanner returned no batch for a non-empty window") + /** + * The tail after the last delimiter, as `Spliterator.#drain` leaves it: never CRLF-trimmed. + */ + #finish(out: string[][]): void { + this.#done = true + + const tailStart = this.#state.cellStartUnits + const tailEnd = this.#text.length + + if (tailStart < tailEnd) { + const value = this.#cell(tailStart, tailEnd, this.#state.cellFlags) + + this.#row[this.#width++] = value + out.push(this.#finishRow()) + } else if (this.#width) { + // A row whose last cell is empty: `a,` at EOF. + const value = this.#cell(tailStart, tailEnd, 0) + + this.#row[this.#width++] = value + out.push(this.#finishRow()) + } else if (!this.#skipEmpty) { + // An empty source, or one ending on a row delimiter: one empty row, matching String.split. + out.push([""]) } + } +} - const cells = scan.cells - - for (let i = 0; i < scan.count; i++) { - const offset = i * CELL_RESULT_STRIDE - const start = cells[offset]! - const cellEnd = cells[offset + 1]! - const flags = cells[offset + 2]! - - // Evaluated before the store: `row[width++] = cell(...)` would bump `width` before `cell` reads it. - const value = cell(start, cellEnd, flags) - - row[width++] = value +/** + * The rows of a wholly in-memory CSV, one non-empty array per kernel batch. The in-memory parser walks these itself so + * each row crosses one generator, its own, rather than two. + */ +export function* scanCsvCellBatches(source: Uint8Array, text: string, init: CsvCellScanInit): Generator { + const scanner = new CellRowScanner(source, text, init) + let batch: string[][] = [] - if (flags & CELL_FLAG_ROW_END) { - // Empty means the raw row range is empty: one cell, and it spans nothing after CRLF removal. A row's first - // cell starts where the row does, so its raw start (before any BOM strip) is the row's start. - const empty = width === 1 && cellEnd === start - const finished = finishRow() + while (scanner.nextBatch(batch)) { + if (batch.length) { + yield batch - if (!(empty && skipEmpty)) { - yield finished - } - } + batch = [] } - - state = scan } - // The tail after the last delimiter, as `Spliterator.#drain` leaves it: never CRLF-trimmed. - const tailStart = state.cellStartUnits - const tailEnd = text.length + if (batch.length) { + yield batch + } +} - if (tailStart < tailEnd) { - const value = cell(tailStart, tailEnd, state.cellFlags) +/** + * {@linkcode scanCsvCellBatches} flattened to one row per step. The parity reference for tests and the shape the bulk + * parser of small sources returns. + */ +export function* scanCsvCells(source: Uint8Array, text: string, init: CsvCellScanInit): Generator { + for (const batch of scanCsvCellBatches(source, text, init)) { + yield* batch + } +} - row[width++] = value - yield finishRow() - } else if (width) { - // A row whose last cell is empty: `a,` at EOF. - const value = cell(tailStart, tailEnd, 0) +/** + * Every row of `source` in one array, pushed straight from the scanner: the streaming path's per-window shape. + */ +export function collectCsvCells(source: Uint8Array, text: string, init: CsvCellScanInit): string[][] { + const scanner = new CellRowScanner(source, text, init) + const rows: string[][] = [] - row[width++] = value - yield finishRow() - } else if (!skipEmpty) { - // An empty source, or one ending on a row delimiter: one empty row, matching String.split. - yield [""] + while (scanner.nextBatch(rows)) { + // Each call appends its batch. } + + return rows } /** @@ -272,7 +373,7 @@ async function* windows( const window = result.value const text = decodeForCellScan(window) - yield text === null ? fallback(window) : Array.from(scanCsvCells(window, text, init)) + yield text === null ? fallback(window) : collectCsvCells(window, text, init) } } finally { await engine.return() From a0f8b7e5caa33a7fc29827fb6a6dcec2a9f9b839 Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:38:51 +0200 Subject: [PATCH 05/10] Decode streamed text and JSON rows per engine window TextSpliterator.fromAsync and JSONSpliterator.fromAsync hand a streamed engine to windowedTextRows, which decodes each window once, splits the text on the delimiter (crlf and skipEmpty applied there), and yields the rows as a batched source. One decode and one await per high-water mark of rows instead of one of each per row. drop and take become sequence ops on these paths. A fatal decode failure falls back to per-row decoding so the error names the row; a text comment filter mirrors the byte one. 600k JSONL rows: 1289ms -> 707ms, level with readline + JSON.parse (702ms); a 500MB corpus file 1948ms against readline's 2092ms. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- AGENTS.md | 13 +-- README.md | 12 +-- lib/formats/JSONSpliterator.ts | 67 ++++++++++--- lib/formats/TextSpliterator.ts | 58 +++++++++-- lib/formats/comment-filter.ts | 33 +++++++ lib/io/windowed-text.ts | 117 +++++++++++++++++++++++ test/io/windowed-text.test.ts | 169 +++++++++++++++++++++++++++++++++ 7 files changed, 434 insertions(+), 35 deletions(-) create mode 100644 lib/io/windowed-text.ts create mode 100644 test/io/windowed-text.test.ts diff --git a/AGENTS.md b/AGENTS.md index edbf6cc..258da87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -190,6 +190,7 @@ Every `fromAsync` opens its source through `openDelimitedRows`, which reads sour - **A raw synchronous parse looks far better than this path can deliver** (~1.6× even at 1GiB). The rest is eaten by the per-row cost of `AsyncSequence`, which both paths pay. Don't re-derive the raw number and conclude the threshold should be raised. - **The bulk path awaits `CharacterSequence.whenReady()`.** The sync engine normally misses the WASM scanner because it finishes before the module loads; reaching it through an async path is the one place that can be fixed, worth ~26% on a large source. `CSVSpliterator.fromAsync` awaits it on the streaming branch too, in `openRows`, because the windowed cell scan below checks `hasScanner()` and a fresh process would otherwise take the row path for its first file. - **Streaming CSV scans cells per engine window.** `AsyncSpliterator.nextWindow()` dequeues every queued range at once and returns the contiguous view from the first range's start to the last range's end, whole records only, about a high-water mark (64 KiB) of rows per `await`. `scanCsvCellsStreaming` (`lib/formats/csv-cells.ts`) decodes each window with the fatal decoder and runs the cell scanner over it (`collectCsvCells`, every row of the window pushed into one array); nothing carries across windows because the engine cut the records with quote state in hand. A window that is not UTF-8 takes the per-row path for that window alone. The rows go out as a batched source (see `AsyncSequence`); yielding them singly was slower than the per-row path. `columnScan: "rows"` opts out, and it remains the parity reference. The engine's `skipEmpty`/`drop`/`take` do not apply to windows; the CSV path applies all three itself. +- **Streamed text and JSON rows are decoded per engine window too.** `TextSpliterator.fromAsync` and `JSONSpliterator.fromAsync` hand a streamed `AsyncSpliterator` to `windowedTextRows` (`lib/io/windowed-text.ts`), which decodes each `nextWindow()` once and splits the text on the delimiter, honouring `crlf` and `skipEmpty` itself, and hands the rows out as a batched source. A `TextDecoder.decode` call per row costs a few hundred nanoseconds of fixed overhead in Node, and the per-row `await` the rest: 600k JSONL rows went 1289ms → 707ms, level with `readline` + `JSON.parse` (702ms), where this path had been half its speed; a 500MB corpus file 1948ms against `readline`'s 2092ms. `drop` and `take` are sequence ops on these two paths rather than engine options, since a window cannot have rows removed from it; they still count after `skipEmpty`. A window that fails a `fatal` decode is decoded row by row so the error names the row. The bulk (small-file) branch is unchanged and still decodes per row. - **Unsized sources use an end-of-input test, not a size test.** Pull one chunk; if the stream is already exhausted the whole input is in hand. Otherwise the pulled chunks are put back in front via a re-headed iterable — `test/io/adaptive-source.test.ts` covers 1/7/64/1024-byte chunkings because losing a chunk here would be silent. - **`bulkThreshold: 0` forces streaming.** Use it when a bounded footprint is the point. - Both engines must agree exactly; the parity tests assert the same fixture through both. @@ -198,12 +199,12 @@ Every `fromAsync` opens its source through `openDelimitedRows`, which reads sour Keyed off **per-row work**, not file size: -| Per-row work | Dominates | Use | -| ----------------------------------------- | ----------- | -------------------------------------------------------------------------- | -| None (count, segment, extract a field) | The scan | `Spliterator` raw ranges + SIMD (~5–6 GB/s vs ~600 MB/s JS) | -| ~1–3 µs (`JSON.parse`, CSV→object) | The parse | Sequential `fromAsync`. Threads lose (0.3–0.9×); JSONL ~0.5× of `readline` | -| Milliseconds (inference, geocode, crypto) | The handler | `parallelMapWorkers` / `asManyWorkers` | -| I/O-bound (file fan-out, network) | Latency | `parallelMap`. Peaks ~2–3 concurrency, then degrades | +| Per-row work | Dominates | Use | +| ----------------------------------------- | ----------- | --------------------------------------------------------------------------------- | +| None (count, segment, extract a field) | The scan | `Spliterator` raw ranges + SIMD (~5–6 GB/s vs ~600 MB/s JS) | +| ~1–3 µs (`JSON.parse`, CSV→object) | The parse | Sequential `fromAsync`. Threads lose (0.3–0.9×); JSONL about even with `readline` | +| Milliseconds (inference, geocode, crypto) | The handler | `parallelMapWorkers` / `asManyWorkers` | +| I/O-bound (file fan-out, network) | Latency | `parallelMap`. Peaks ~2–3 concurrency, then degrades | Naming rule: **closure ⇒ caller's thread; module path ⇒ worker thread** (closures can't cross `postMessage`). diff --git a/README.md b/README.md index f195090..4b94b43 100644 --- a/README.md +++ b/README.md @@ -297,12 +297,12 @@ For information on all available commands, run `spliterator --help`. The question that predicts the answer is not "how big is my file?" — it's **how much work happens per row.** -| Per-row work | What dominates | Reach for | -| ------------------------------------------------------------ | -------------- | --------------------------------------------------------------------------------------------------- | -| None — counting, segmenting, pulling a couple of fields | The scan | `Spliterator` raw byte ranges. The SIMD scanner earns its keep here (~5–6 GB/s vs ~600 MB/s for JS) | -| ~1–3 µs — `JSON.parse`, CSV → object, string normalize | The parse | Plain sequential `fromAsync`. Threads **lose** here (0.3–0.9×); JSONL runs ~0.5× of `readline` | -| Milliseconds — model inference, geocoding, crypto, image ops | Your handler | `parallelMapWorkers`, or `AsyncSpliterator.asManyWorkers` for one large file | -| I/O-bound — file fan-out, network | Latency | `parallelMap` (caller's thread). Concurrency peaks around 2–3, then **degrades** | +| Per-row work | What dominates | Reach for | +| ------------------------------------------------------------ | -------------- | ----------------------------------------------------------------------------------------------------- | +| None — counting, segmenting, pulling a couple of fields | The scan | `Spliterator` raw byte ranges. The SIMD scanner earns its keep here (~5–6 GB/s vs ~600 MB/s for JS) | +| ~1–3 µs — `JSON.parse`, CSV → object, string normalize | The parse | Plain sequential `fromAsync`. Threads **lose** here (0.3–0.9×); JSONL runs about even with `readline` | +| Milliseconds — model inference, geocoding, crypto, image ops | Your handler | `parallelMapWorkers`, or `AsyncSpliterator.asManyWorkers` for one large file | +| I/O-bound — file fan-out, network | Latency | `parallelMap` (caller's thread). Concurrency peaks around 2–3, then **degrades** | The line worth internalizing: **the scan is almost never your bottleneck unless you aren't parsing.** Measure before adopting a parallel primitive. diff --git a/lib/formats/JSONSpliterator.ts b/lib/formats/JSONSpliterator.ts index 6d949ff..709f995 100644 --- a/lib/formats/JSONSpliterator.ts +++ b/lib/formats/JSONSpliterator.ts @@ -4,11 +4,13 @@ * @copyright Sister Software */ -import type { CharacterSequenceInput } from "#core/CharacterSequence" +import { AsyncSpliterator } from "#core/AsyncSpliterator" +import { CharacterSequence, type CharacterSequenceInput } from "#core/CharacterSequence" import { Spliterator, type SpliteratorInit } from "#core/Spliterator" -import { type CommentInput, createCommentFilter } from "#formats/comment-filter" +import { type CommentInput, createCommentFilter, createTextCommentFilter } from "#formats/comment-filter" import type { AsyncDataResource } from "#internal/shared" import { type AdaptiveSourceInit, openDelimitedRows } from "#io/adaptive-source" +import { windowedTextRows } from "#io/windowed-text" import { AsyncSequence } from "#iterators/AsyncSequence" import { Sequence } from "#iterators/Sequence" @@ -72,11 +74,12 @@ function* parseRows( * row still throws. The check is a prefix test rather than a recovery from `JSON.parse`, so corrupt data cannot be * mistaken for a header. * - * **Performance caveat.** This path is `JSON.parse`-bound rather than scan-bound. Measured over 500k rows (88MB, ~177B - * per row, Node 26): the delimiter scan is ~140ms and decoding ~46ms, while `JSON.parse` and the per-row async - * machinery account for the rest of ~1280ms. Against Node's `readline` + `JSON.parse` at ~630ms, this runs **roughly - * half the speed, which is a net loss.** Use it for API convenience, or when streaming to bound memory. Do not swap a - * working `readline` loop to it expecting a speedup on parse-heavy JSONL. + * **Performance.** This path is `JSON.parse`-bound rather than scan-bound, so it runs about even with Node's `readline` + * plus `JSON.parse` rather than ahead of it. A streamed source is decoded per engine window and split as text + * (`windowedTextRows`), one decode and one `await` per high-water mark of rows instead of one of each per row: measured + * on 600k rows of 120MB (Node 26), 1289ms per row against 707ms windowed, with `readline` + `JSON.parse` at 702ms; on a + * 500MB corpus file of 1.27M rows, 1948ms against `readline`'s 2092ms. Reach for it for the API and the bounded + * footprint, not for a speedup on parse-heavy JSONL. * * The scan advantage only shows when you _don't_ fully parse every row. Filtering on the raw text and parsing only what * survives is the shape that wins — parsing 20% of rows measured ~2.5× faster than parsing all of them: @@ -109,22 +112,56 @@ export abstract class JSONSpliterator { */ public static fromAsync( source: AsyncDataResource, - { comment, ...options }: AdaptiveSourceInit & JSONSpliteratorInit = {} + { comment, drop = 0, take = Infinity, ...options }: AdaptiveSourceInit & JSONSpliteratorInit = {} ): AsyncSequence { const decoder = new TextDecoder() - const parseable = createCommentFilter(comment) + const parseableBytes = createCommentFilter(comment) + const parseableText = createTextCommentFilter(comment) + + // A streamed source is decoded per engine window and split as text (see `windowedTextRows`), which is where this + // path stopped losing to `readline`: 1289ms → 707ms over 600k rows, `readline` + `JSON.parse` at 702ms. The bulk + // branch's rows are bytes decoded per row below. `drop` and `take` are sequence ops rather than engine options + // because a window cannot have rows removed from it, and they count after `skipEmpty` either way. + let rows = AsyncSequence.from(async () => { + const opened = await openDelimitedRows(source, options) + + return opened instanceof AsyncSpliterator + ? windowedTextRows(opened, { + delimiter: new CharacterSequence(options.delimiter), + crlf: options.crlf ?? false, + skipEmpty: options.skipEmpty ?? true, + decoder, + decodeError: (rowIndex, cause) => { + const error = new SyntaxError(`Failed to decode data at row ${rowIndex}`) + error.cause = cause + + return error + }, + }) + : opened + }) - // Parsing is an op on the sequence rather than a generator wrapped inside one. An allocating row body makes the extra - // async frame a wrapping generator adds disproportionately expensive: 1460ms against 1278ms over 500k rows, where - // the same layer costs a third as much when the body only decodes. - const rows = AsyncSequence.from(() => openDelimitedRows(source, options)) + if (drop > 0) { + rows = rows.drop(drop) + } + + if (Number.isFinite(take)) { + rows = rows.take(take) + } // Skipping is a fused op ahead of the parse. The chain runs its ops in one loop, so // this costs an extra iteration per row rather than an extra async boundary. When no prefix is configured, the // chain adds no operation and the hot path is unchanged. - return (parseable ? rows.filter(parseable) : rows).map((row, rowCursor) => { + if (parseableBytes && parseableText) { + rows = rows.filter((row) => (typeof row === "string" ? parseableText(row) : parseableBytes(row))) + } + + // Parsing is an op on the sequence rather than a generator wrapped inside one. An allocating row body makes the extra + // async frame a wrapping generator adds disproportionately expensive: 1460ms against 1278ms over 500k rows, where + // the same layer costs a third as much when the body only decodes. + return rows.map((row, rowCursor) => { try { - return JSON.parse(decoder.decode(row)) as T + return JSON.parse(typeof row === "string" ? row : decoder.decode(row)) as T } catch (parsedError) { const error = new SyntaxError(`Failed to parse JSON at row ${rowCursor}`) error.cause = parsedError diff --git a/lib/formats/TextSpliterator.ts b/lib/formats/TextSpliterator.ts index bdb34f1..ed5585a 100644 --- a/lib/formats/TextSpliterator.ts +++ b/lib/formats/TextSpliterator.ts @@ -4,10 +4,12 @@ * @copyright Sister Software */ -import type { CharacterSequenceInput } from "#core/CharacterSequence" +import { AsyncSpliterator } from "#core/AsyncSpliterator" +import { CharacterSequence, type CharacterSequenceInput } from "#core/CharacterSequence" import { Spliterator, type SpliteratorInit } from "#core/Spliterator" import type { AsyncDataResource } from "#internal/shared" import { type AdaptiveSourceInit, openDelimitedRows } from "#io/adaptive-source" +import { windowedTextRows } from "#io/windowed-text" import { AsyncSequence } from "#iterators/AsyncSequence" import { Sequence } from "#iterators/Sequence" @@ -181,23 +183,63 @@ export abstract class TextSpliterator { */ public static fromAsync( source: AsyncDataResource, - { encoding, fatal, ignoreBOM, trim = true, ...options }: TextSpliteratorInit & AdaptiveSourceInit = {} + { + encoding, + fatal, + ignoreBOM, + trim = true, + drop = 0, + take = Infinity, + ...options + }: TextSpliteratorInit & AdaptiveSourceInit = {} ): AsyncSequence { const decoder = new TextDecoder(encoding, { fatal, ignoreBOM }) - const dropBlank = trim && (options.skipEmpty ?? true) + const skipEmpty = options.skipEmpty ?? true + const dropBlank = trim && skipEmpty + + const decodeError = (rowIndex: number, cause: unknown) => { + const error = new SyntaxError(`Failed to decode data at row ${rowIndex}`) + error.cause = cause + + return error + } + + // A streamed source is decoded per engine window and split as text (see `windowedTextRows`); the bulk branch's + // rows are bytes decoded per row below. `drop` and `take` are sequence ops rather than engine options because a + // window cannot have rows removed from it, and they count after `skipEmpty` either way. + let rows = AsyncSequence.from(async () => { + const opened = await openDelimitedRows(source, options) + + return opened instanceof AsyncSpliterator + ? windowedTextRows(opened, { + delimiter: new CharacterSequence(options.delimiter), + crlf: options.crlf ?? false, + skipEmpty, + decoder, + decodeError, + }) + : opened + }) + + if (drop > 0) { + rows = rows.drop(drop) + } + + if (Number.isFinite(take)) { + rows = rows.take(take) + } // Decoding is an op on the sequence rather than a generator wrapped inside one. A wrapping generator adds an async frame // per row on top of the sequence's own, which measured 297ms against 279ms over 500k rows. - const decoded = AsyncSequence.from(() => openDelimitedRows(source, options)).map((row, rowCursor) => { + const decoded = rows.map((row, rowCursor) => { + if (typeof row === "string") return trim ? row.trim() : row + try { const text = decoder.decode(row) return trim ? text.trim() : text } catch (parsedError) { - const error = new SyntaxError(`Failed to decode data at row ${rowCursor}`) - error.cause = parsedError - - throw error + throw decodeError(rowCursor, parsedError) } }) diff --git a/lib/formats/comment-filter.ts b/lib/formats/comment-filter.ts index fce9f65..cdf6285 100644 --- a/lib/formats/comment-filter.ts +++ b/lib/formats/comment-filter.ts @@ -43,6 +43,39 @@ export type RowPredicate = (row: Uint8Array) => boolean * worker whose range opens inside a block comment has no way to know. A line prefix is self-describing at every row * boundary, which is why every JSONL-with-comments convention in the wild is one. */ +export type TextRowPredicate = (row: string) => boolean + +/** + * {@linkcode createCommentFilter} for rows that are already text, which is what the windowed streaming path hands out. + * Same semantics: a blank row and a row whose first non-whitespace characters are a prefix are dropped. + */ +export function createTextCommentFilter(comment: CommentInput | undefined): TextRowPredicate | null { + if (comment === undefined) return null + + const prefixes = (typeof comment === "string" ? [comment] : comment).filter((prefix) => prefix.length > 0) + + return (row) => { + const { length } = row + let cursor = 0 + + while (cursor < length) { + const code = row.charCodeAt(cursor) + + if (code !== SPACE && code !== HORIZONTAL_TAB && code !== CARRIAGE_RETURN && code !== LINE_FEED) break + + cursor++ + } + + if (cursor === length) return false + + for (const prefix of prefixes) { + if (row.startsWith(prefix, cursor)) return false + } + + return true + } +} + export function createCommentFilter(comment: CommentInput | undefined): RowPredicate | null { if (comment === undefined) return null diff --git a/lib/io/windowed-text.ts b/lib/io/windowed-text.ts new file mode 100644 index 0000000..c0f96eb --- /dev/null +++ b/lib/io/windowed-text.ts @@ -0,0 +1,117 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import type { AsyncSpliterator } from "#core/AsyncSpliterator" +import { type CharacterSequence, Delimiters } from "#core/CharacterSequence" +import { Spliterator } from "#core/Spliterator" +import { batched, type BatchedAsyncIterable } from "#iterators/AsyncSequence" + +export interface WindowedTextInit { + /** + * The delimiter as the engine was given it. Decoded once with `decoder` so the window's text splits on the same + * characters the engine split its bytes on. + */ + delimiter: CharacterSequence + crlf: boolean + skipEmpty: boolean + decoder: TextDecoder + /** + * Called with the row index of a window that failed to decode, after the rows before it were handed out. The row + * path's decode error names a row, so the window path finds which one by decoding that window row by row. + */ + decodeError: (rowIndex: number, cause: unknown) => Error +} + +/** + * Decode a streaming source one engine window at a time and split the text into rows, handed out as batches. + * + * The per-row path decodes each row's bytes on its own: a `TextDecoder.decode` call per row, which has a fixed cost of + * a few hundred nanoseconds in Node, plus an `await` per row through the sequence. One decode per window (a high-water + * mark of rows) and one `await` per window replaces both, and the rows come out as slices of the window's text. + * Measured on a 120MB JSONL file of 600k rows: `JSONSpliterator.fromAsync` 1289ms per row against 584ms through a + * window, with `readline` + `JSON.parse` at 783ms. The same split as the CSV cell scanner's streaming path, without the + * kernel. + * + * The engine's `skipEmpty`, `drop` and `take` do not apply to windows, so `skipEmpty` is applied here and the caller + * applies `drop` and `take` as sequence ops. A window that fails to decode (a `fatal` decoder) is decoded row by row so + * the error can name the row, as the per-row path's would have. + * + * `return()` on the iterable closes the engine. + */ +export function windowedTextRows(engine: AsyncSpliterator, init: WindowedTextInit): BatchedAsyncIterable { + return batched(windows(engine, init)) +} + +async function* windows(engine: AsyncSpliterator, init: WindowedTextInit): AsyncGenerator { + const { crlf, skipEmpty, decoder } = init + const delimiterText = decoder.decode(init.delimiter) + const delimiterLength = delimiterText.length + let rowIndex = 0 + + try { + while (true) { + const result = await engine.nextWindow() + + if (result.done) return + + const window = result.value + let text: string + + try { + text = decoder.decode(window) + } catch { + // Find the row: decode the window the way the row path would have, so the error names the same row. + const rows: string[] = [] + + for (const row of Spliterator.fromSync(window, { delimiter: init.delimiter, crlf, skipEmpty })) { + try { + rows.push(decoder.decode(row)) + } catch (error) { + throw init.decodeError(rowIndex + rows.length, error) + } + } + + rowIndex += rows.length + + yield rows + + continue + } + + const rows: string[] = [] + let position = 0 + + // The window holds whole records with the delimiters between them and none after the last, exactly as the + // engine cut them: the piece after the last delimiter is the last record, not a partial one. + while (true) { + const found = text.indexOf(delimiterText, position) + let end = found === -1 ? text.length : found + + // A carriage return before a delimiter is part of the delimiter; one at the end of the input is data, and the + // engine already trimmed the one before the last delimiter it cut. + if (crlf && found !== -1 && end > position && text.charCodeAt(end - 1) === Delimiters.CarriageReturn) { + end-- + } + + if (!(skipEmpty && end === position)) { + rows.push(text.slice(position, end)) + } + + if (found === -1) break + + position = found + delimiterLength + } + + rowIndex += rows.length + + if (rows.length) { + yield rows + } + } + } finally { + await engine.return() + } +} diff --git a/test/io/windowed-text.test.ts b/test/io/windowed-text.test.ts new file mode 100644 index 0000000..da7445f --- /dev/null +++ b/test/io/windowed-text.test.ts @@ -0,0 +1,169 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { JSONSpliterator, TextSpliterator } from "spliterator" +import { describe, expect, test } from "vitest" + +/** + * The streamed branch of `TextSpliterator.fromAsync` and `JSONSpliterator.fromAsync` decodes per engine window and + * splits the text; the sync `from` decodes per row. Every case here runs the streamed branch at a tiny high-water mark, + * so a row straddles windows, and checks it against the sync result. + */ +const STREAM = { bulkThreshold: 0, highWaterMark: 48 } as const + +function source(text: string) { + const bytes = new TextEncoder().encode(text) + + return { + async *[Symbol.asyncIterator]() { + // Chunks of 7 bytes, so multi-byte characters and delimiters straddle chunk edges too. + for (let offset = 0; offset < bytes.length; offset += 7) { + yield bytes.subarray(offset, offset + 7) + } + }, + } +} + +const texts: Array<[string, string]> = [ + ["plain", "one\ntwo\nthree\n"], + ["no trailing newline", "one\ntwo\nthree"], + ["blank rows", "one\n\n \ntwo\n\n"], + ["crlf", "one\r\ntwo\r\nthree\r\n"], + ["lone cr at end of input", "one\ntwo\r"], + ["utf-8 across windows", "é한😀 one\nzwei 한국어 긴 줄입니다 정말로\n三\n"], + ["long rows past the window", `${"x".repeat(200)}\n${"y".repeat(100)}\nz\n`], + ["empty", ""], + ["only newlines", "\n\n\n"], +] + +describe("windowed text rows", () => { + for (const [label, text] of texts) { + test(`${label}: streamed matches sync`, async () => { + expect(await TextSpliterator.fromAsync(source(text), STREAM).toArray()).toEqual( + TextSpliterator.from(text).toArray() + ) + }) + + test(`${label}: streamed matches sync with trim off and empties kept`, async () => { + const init = { trim: false, skipEmpty: false } + + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( + TextSpliterator.from(text, init).toArray() + ) + }) + + test(`${label}: streamed matches sync with crlf`, async () => { + const init = { trim: false, crlf: true } + + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( + TextSpliterator.from(text, init).toArray() + ) + }) + } + + test("a multi-character delimiter", async () => { + const text = "a<>b<>c<>" + const init = { delimiter: "<>", trim: false } + + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual(["a", "b", "c"]) + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( + TextSpliterator.from(text, init).toArray() + ) + }) + + test("drop and take count rows after empties are skipped, as the engine's did", async () => { + const text = "a\n\nb\n\nc\nd\ne\n" + const init = { drop: 1, take: 2 } + + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual(["b", "c"]) + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( + TextSpliterator.from(text, init).toArray() + ) + }) + + test("take closes the source", async () => { + let closed = false + const bytes = new TextEncoder().encode("a\nb\nc\nd\n") + + const chunks = { + async *[Symbol.asyncIterator]() { + try { + for (let offset = 0; offset < bytes.length; offset += 2) { + yield bytes.subarray(offset, offset + 2) + } + } finally { + closed = true + } + }, + } + + expect(await TextSpliterator.fromAsync(chunks, { ...STREAM, take: 1 }).toArray()).toEqual(["a"]) + expect(closed).toBe(true) + }) + + test("a fatal decoder names the bad row", async () => { + const bytes = new Uint8Array([...new TextEncoder().encode("ok\nfine\n"), 0xff, 0xfe, 0x0a, 0x7a, 0x0a]) + + const chunks = { + async *[Symbol.asyncIterator]() { + yield bytes + }, + } + + await expect(TextSpliterator.fromAsync(chunks, { ...STREAM, fatal: true }).toArray()).rejects.toThrow( + "Failed to decode data at row 2" + ) + }) + + test("a lossy decoder replaces and continues", async () => { + const bytes = new Uint8Array([...new TextEncoder().encode("ok\n"), 0xff, 0x0a, 0x7a, 0x0a]) + + const chunks = { + async *[Symbol.asyncIterator]() { + yield bytes + }, + } + + expect(await TextSpliterator.fromAsync(chunks, STREAM).toArray()).toEqual(["ok", "�", "z"]) + }) +}) + +describe("windowed JSON rows", () => { + test("streamed matches sync", async () => { + const text = '{"a":1}\n{"b":"한국어"}\n\n{"c":[1,2,3]}\n' + + expect(await JSONSpliterator.fromAsync(source(text), STREAM).toArray()).toEqual( + JSONSpliterator.from(text).toArray() + ) + }) + + test("comment rows and blank rows are skipped on the text path", async () => { + const text = '// header\n{"a":1}\n \n # note\n{"b":2}\n' + const init = { comment: ["//", "#"] } + + expect(await JSONSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual([ + { a: 1 }, + { b: 2 }, + ]) + expect(await JSONSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( + JSONSpliterator.from(text, init).toArray() + ) + }) + + test("a bad row names its index", async () => { + const text = '{"a":1}\n{"b":2}\nnot json\n' + + await expect(JSONSpliterator.fromAsync(source(text), STREAM).toArray()).rejects.toThrow( + "Failed to parse JSON at row 2" + ) + }) + + test("drop and take", async () => { + const text = "1\n2\n3\n4\n5\n" + + expect(await JSONSpliterator.fromAsync(source(text), { ...STREAM, drop: 1, take: 3 }).toArray()).toEqual([2, 3, 4]) + }) +}) From bf29a6c2dafe72f16ecdb5b7fda58f35fa082503 Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:47:55 +0200 Subject: [PATCH 06/10] Hand the cell kernel's batch to the scanner as a view CharacterSequence.scanCells returns the result block as a window-relative view with a unitBase instead of copying it out and rebasing in a JS loop. CellRowScanner.nextBatch turns the whole batch into rows before returning and runs no user code, so nothing can scan in between. 386MB streamed into objects 1869ms -> 1685ms, litmus in-memory 8.3ms -> 7.4ms, 300MB quoted stream 1.9s -> 1.8s. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- AGENTS.md | 2 +- lib/core/CharacterSequence.ts | 22 +++++++++------------- lib/core/wasm_module.ts | 9 +++++++-- lib/formats/csv-cells.ts | 7 +++++-- test/core/wasm.test.ts | 17 +++++++++++------ 5 files changed, 33 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 258da87..88d0661 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -317,7 +317,7 @@ The parallel-parsing layers are tested bottom-up so the worker protocol is verif - **A string source is the decoded text**: `CSVSpliterator.from(string)` passes the string itself to the cell scanner instead of decoding the bytes it encoded from it. The offsets agree because a lone surrogate encodes as U+FFFD, one UTF-16 unit either way, so the surrogate survives where the row path yields U+FFFD. Worth 11% (4.2s → 3.8s) and 770 MiB of peak memory on a 386MB string, the second copy of the source; retained cells then keep the caller's string alive rather than a copy. `columnScan: "rows"` no longer saves memory on a string source for that reason: it decodes per row and its cells are fresh strings, 1.8 GiB against 1.5 GiB retained. - **`unquoteColumn` only calls `replaceAll` after something says a doubled quote is there**: the row-path walk `splitQuotedString` carries its own `sawDoubledQuote` and unquotes as it splits rather than slicing each quoted cell twice; `unquoteColumn` itself runs one `indexOf('""')`. The cell-scan path never calls it: the kernel's `CELL_FLAG_HAS_ESCAPE` (a quote that opened directly after a quote, carried across windows through `previous_byte` and the state's `cell_flags`) says whether to unescape, and the cell is sliced from the source text once, inside the quotes. The two-pass form plus unconditional `replaceAll` was 17% of a streamed quoted parse and 30% of the quoted walk; the kernel flag was another 5% on the stream. All three must keep agreeing on what a quoted field means. - **The cell-scan path is scavenger-bound when rows are retained, so its garbage per cell matters more than its instructions**: on uDSV's 1.9MB `litmus_quoted` fixture the parse ran at a steady 9ms when rows were discarded and 19ms when kept, because it allocated ~23MB per parse against ~8.5MB for uDSV, so V8 scavenged once per parse instead of once per three and each scavenge copied the ~12MB result built so far (`--trace-gc-nvp`: `allocated`, `new_space_survived`). Two allocations per cell were the difference: the quoted outer string sliced before the inner one, and rows grown by `push`, which left a 43-slot store behind 20 cells plus two discarded stores per row. Cells are now sliced once and rows are filled into an exact-size `template.slice()` the width of the previous row. Harness-style geometric mean 17.6ms → 8.7ms (103 → ~210 MiB/s), GC 6.9ms → 0.9ms per parse, and the 300MB quoted stream 2.4s → 2.1s. When a change here looks free in a discard-rows loop, measure it with the rows kept. -- **Kernel result views alias shared memory**: `CharacterSequence.scanCells` copies each batch out before returning, because a transformer may run a nested parse while the outer row is being assembled. Don't "optimize" the copy away. +- **Kernel result views alias shared memory**: `CharacterSequence.scanCells` returns a view of the result block, window-relative with a `unitBase` to add, valid only until the next scanner call. That is safe because its one consumer, `CellRowScanner.nextBatch`, turns the whole batch into rows before returning and runs no user code; transformers and emitters run later, on the rows. It used to copy and rebase the batch in a JS loop, 9% of a streamed parse. Any new consumer that yields or calls out mid-batch must copy first. - **A static `import` of anything under `out/` from a test breaks the second `tsc -b`** (TS5055, the emitted `.d.ts` becomes its own input). Tests that need un-exported internals load them in a `beforeAll` through a dynamic import whose specifier is a variable; see `test/core/wasm.test.ts`. - **`Array.shift()` is O(n)**: Avoid `shift()` on large arrays in hot paths. Use a `head` pointer instead (`chunks[head++]`). diff --git a/lib/core/CharacterSequence.ts b/lib/core/CharacterSequence.ts index 3d05be6..1cca8ef 100644 --- a/lib/core/CharacterSequence.ts +++ b/lib/core/CharacterSequence.ts @@ -423,12 +423,14 @@ export class CharacterSequence extends Uint8Array { } /** - * Scan CSV cells over `[state.scanCursor, end)` in one bounded kernel call, returning an owned batch rebased to - * absolute byte and UTF-16 offsets. The kernel counts UTF-16 units as it scans, which is what lets the caller slice a - * decoded string by these offsets without an ASCII gate; the decode must be `fatal` so the count is exact. + * Scan CSV cells over `[state.scanCursor, end)` in one bounded kernel call. The kernel counts UTF-16 units as it + * scans, which is what lets the caller slice a decoded string by these offsets without an ASCII gate; the decode must + * be `fatal` so the count is exact. * - * The batch is copied out of WASM memory before returning, so a caller may hold it across further scans, including a - * nested parse run by user code while a row is being consumed. + * `cells` is a view of shared WASM memory, in units relative to `unitBase`, valid until the next scanner call of any + * kind. The one consumer, `CellRowScanner`, turns the whole batch into rows before it returns, and nothing runs user + * code in between, so the copy and the rebasing loop that used to be here were 9% of a streamed parse for nothing. + * The state fields are absolute. * * Returns `null` when the scanner is unavailable or the window is empty. Callers keep their own fallback. */ @@ -480,17 +482,11 @@ export class CharacterSequence extends Uint8Array { ) const block = new Int32Array(wasm.memory.buffer, resultsOffset, CELL_RESULT_HEADER + count * CELL_RESULT_STRIDE) - // Copy: the view aliases shared memory that the next scanner call overwrites. - const cells = block.slice(CELL_RESULT_HEADER) const unitBase = state.units - for (let i = 0; i < count; i++) { - cells[i * CELL_RESULT_STRIDE] = cells[i * CELL_RESULT_STRIDE]! + unitBase - cells[i * CELL_RESULT_STRIDE + 1] = cells[i * CELL_RESULT_STRIDE + 1]! + unitBase - } - return { - cells, + cells: block.subarray(CELL_RESULT_HEADER), + unitBase, count, scanCursor: windowStart + block[0]!, units: unitBase + block[1]!, diff --git a/lib/core/wasm_module.ts b/lib/core/wasm_module.ts index c335081..e5242a1 100644 --- a/lib/core/wasm_module.ts +++ b/lib/core/wasm_module.ts @@ -149,13 +149,18 @@ export interface WasmRangeScanResult { } /** - * One owned batch from the CSV cell scanner, rebased to absolute offsets by `CharacterSequence.scanCells`. + * One batch from the CSV cell scanner, as `CharacterSequence.scanCells` returns it. */ export interface WasmCellScanResult { /** - * `[start, end, flags]` triples in UTF-16 units of the decoded source. A copy, safe to hold across further scans. + * `[start, end, flags]` triples, `start` and `end` in UTF-16 units relative to `unitBase`. A view of shared WASM + * memory: consume it before the next scanner call. */ cells: Int32Array + /** + * Absolute UTF-16 units at the window's start; add it to each cell's `start` and `end`. + */ + unitBase: number count: number /** * Absolute byte offset the scan stopped at. diff --git a/lib/formats/csv-cells.ts b/lib/formats/csv-cells.ts index 5e68000..eb13211 100644 --- a/lib/formats/csv-cells.ts +++ b/lib/formats/csv-cells.ts @@ -123,13 +123,16 @@ export class CellRowScanner { throw new Error("CellRowScanner: the cell scanner returned no batch for a non-empty window") } + // A view of the kernel's memory, relative to `unitBase`: every cell is turned into a row below before anything can + // scan again, and no user code runs in here. const cells = scan.cells + const unitBase = scan.unitBase const skipEmpty = this.#skipEmpty for (let i = 0; i < scan.count; i++) { const offset = i * CELL_RESULT_STRIDE - const start = cells[offset]! - const cellEnd = cells[offset + 1]! + const start = cells[offset]! + unitBase + const cellEnd = cells[offset + 1]! + unitBase const flags = cells[offset + 2]! // Evaluated before the store: `row[width++] = cell(...)` would bump `width` before `cell` reads it. diff --git a/test/core/wasm.test.ts b/test/core/wasm.test.ts index c92e2a9..512c3bd 100644 --- a/test/core/wasm.test.ts +++ b/test/core/wasm.test.ts @@ -700,11 +700,12 @@ describe("WASM SIMD scanner", () => { const options = { rowDelimiter: 0x0a, columnDelimiter: 0x2c, quote: 0x22, crlf: true } const initial = { scanCursor: 0, units: 0, insideQuotes: false, cellStartUnits: 0, cellFlags: 0 } - test("rebases a window's cells and state to absolute offsets", () => { + test("returns a window's cells relative to its unit base and the state in absolute offsets", () => { const bytes = encoder.encode("é,a\nbb,c") // Scan the first row only, then resume. const first = CharacterSequence.scanCells(bytes, initial, 5, options)! + expect(first.unitBase).toBe(0) expect(Array.from(first.cells)).toEqual([0, 1, 0, 2, 3, CELL_FLAG_ROW_END]) expect(first).toMatchObject({ @@ -717,7 +718,8 @@ describe("WASM SIMD scanner", () => { const second = CharacterSequence.scanCells(bytes, first, bytes.length, options)! - expect(Array.from(second.cells)).toEqual([4, 6, 0]) + expect(second.unitBase).toBe(4) + expect(Array.from(second.cells)).toEqual([0, 2, 0]) expect(second).toMatchObject({ scanCursor: 9, units: 8, cellStartUnits: 7 }) }) @@ -737,7 +739,9 @@ describe("WASM SIMD scanner", () => { const second = CharacterSequence.scanCells(bytes, first, bytes.length, options)! - expect(Array.from(second.cells)).toEqual([2, 7, CELL_FLAG_HAS_QUOTE, 8, 9, CELL_FLAG_ROW_END]) + // The open cell began two units before this window, so its relative start is negative. + expect(second.unitBase).toBe(4) + expect(Array.from(second.cells)).toEqual([-2, 3, CELL_FLAG_HAS_QUOTE, 4, 5, CELL_FLAG_ROW_END]) }) test("supplies previous_byte so a CRLF split by the window is trimmed", () => { @@ -748,17 +752,18 @@ describe("WASM SIMD scanner", () => { const second = CharacterSequence.scanCells(bytes, first, bytes.length, options)! - expect(Array.from(second.cells)).toEqual([0, 2, CELL_FLAG_ROW_END, 4, 6, CELL_FLAG_ROW_END]) + expect(second.unitBase).toBe(3) + expect(Array.from(second.cells)).toEqual([-3, -1, CELL_FLAG_ROW_END, 1, 3, CELL_FLAG_ROW_END]) }) - test("returns an owned copy that survives a later scan", () => { + test("the batch is a view the next scan overwrites, so consumers finish it first", () => { const bytes = encoder.encode("a,b\n") const result = CharacterSequence.scanCells(bytes, initial, bytes.length, options)! const snapshot = Array.from(result.cells) CharacterSequence.scanCells(encoder.encode("zzzzzzzz,yyyyyyyy\n"), initial, 18, options) - expect(Array.from(result.cells)).toEqual(snapshot) + expect(Array.from(result.cells)).not.toEqual(snapshot) }) test("honours maxCells and reports where it stopped", () => { From d6a421ae4bf17845a0eaecd50ebb5874bbff72a3 Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:50:36 +0200 Subject: [PATCH 07/10] Resolve worker entries without import.meta.resolve jiti, which Docusaurus loads its plugins through, transpiles an ESM dependency to CommonJS on the fly: it rewrites import.meta.url and leaves import.meta.resolve in place, and Node then refuses the file with "Cannot use 'import.meta' outside a module". 9.1.0 shipped that form in lib/internal/node-modules.js and broke a consumer's docs build. workerEntryUrl now resolves through createRequire(import.meta.url), which honours the # import map the same way, with the built-ins loaded through process.getBuiltinModule so the root keeps no static node: import. A test over the compiled output refuses any import.meta form but .url under out/lib and out/node. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- lib/internal/node-modules.ts | 11 ++++++- test/io/windowed-text.test.ts | 3 ++ test/package/import-meta.test.ts | 55 ++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 test/package/import-meta.test.ts diff --git a/lib/internal/node-modules.ts b/lib/internal/node-modules.ts index bcc4717..0dbc375 100644 --- a/lib/internal/node-modules.ts +++ b/lib/internal/node-modules.ts @@ -30,9 +30,18 @@ export function loadHidden(specifier: string): Promise { * The file URL of a worker entry named by its `#` import-map specifier, so the same specifier serves the source tree * under the `node` condition and the compiled tree under `default`. Taking the specifier as a parameter keeps a bundler * from chasing it as an asset. + * + * Resolved through `createRequire`, not `import.meta.resolve`. Tools that transpile this package to CommonJS on the fly + * (jiti, which Docusaurus loads its plugins through) rewrite `import.meta.url` and leave `import.meta.resolve` in + * place, and Node then refuses the rewritten file with "Cannot use 'import.meta' outside a module". That took down a + * consumer's docs build on 9.1.0. `require.resolve` honours the package's `#` import map the same way. The built-ins + * come through `process.getBuiltinModule` so this stays synchronous and the root keeps no static `node:` import. */ export function workerEntryUrl(specifier: string): URL { - return new URL(import.meta.resolve(specifier)) + const { createRequire } = process.getBuiltinModule("node:module") + const { pathToFileURL } = process.getBuiltinModule("node:url") + + return pathToFileURL(createRequire(import.meta.url).resolve(specifier)) } const NODE_FS = "spliterator/node/fs" diff --git a/test/io/windowed-text.test.ts b/test/io/windowed-text.test.ts index da7445f..6697b50 100644 --- a/test/io/windowed-text.test.ts +++ b/test/io/windowed-text.test.ts @@ -69,6 +69,7 @@ describe("windowed text rows", () => { const init = { delimiter: "<>", trim: false } expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual(["a", "b", "c"]) + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( TextSpliterator.from(text, init).toArray() ) @@ -79,6 +80,7 @@ describe("windowed text rows", () => { const init = { drop: 1, take: 2 } expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual(["b", "c"]) + expect(await TextSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( TextSpliterator.from(text, init).toArray() ) @@ -148,6 +150,7 @@ describe("windowed JSON rows", () => { { a: 1 }, { b: 2 }, ]) + expect(await JSONSpliterator.fromAsync(source(text), { ...STREAM, ...init }).toArray()).toEqual( JSONSpliterator.from(text, init).toArray() ) diff --git a/test/package/import-meta.test.ts b/test/package/import-meta.test.ts new file mode 100644 index 0000000..2677f6e --- /dev/null +++ b/test/package/import-meta.test.ts @@ -0,0 +1,55 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { readdirSync, readFileSync } from "node:fs" +import { join } from "node:path" +import { fileURLToPath } from "node:url" + +import { describe, expect, test } from "vitest" + +const outDir = fileURLToPath(new URL("../../out/", import.meta.url)) + +function* jsFiles(dir: string): Generator { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name) + + if (entry.isDirectory()) { + yield* jsFiles(path) + } else if (entry.name.endsWith(".js")) { + yield path + } + } +} + +describe("import.meta in the shipped library", () => { + /** + * Jiti, which Docusaurus loads its plugins through, transpiles an ESM dependency to CommonJS on the fly. It rewrites + * `import.meta.url` and leaves every other `import.meta` form in place, and Node then refuses the file with "Cannot + * use 'import.meta' outside a module". spliterator 9.1.0 shipped `import.meta.resolve` in + * `lib/internal/node-modules.js` and broke a consumer's docs build. The CLI is not loaded that way and may keep its + * `import.meta.url`. + */ + test("only import.meta.url appears under out/lib and out/node", () => { + const offending: string[] = [] + + for (const dir of ["lib", "node"]) { + for (const file of jsFiles(join(outDir, dir))) { + // Comments may name the forbidden form while explaining why it is forbidden. + const source = readFileSync(file, "utf8") + .replaceAll(/\/\*[\s\S]*?\*\//g, "") + .replaceAll(/^\s*\/\/.*$/gm, "") + + for (const match of source.matchAll(/import\.meta(?:\.(\w+))?/g)) { + if (match[1] !== "url") { + offending.push(`${file.slice(outDir.length)}: ${match[0]}`) + } + } + } + } + + expect(offending).toEqual([]) + }) +}) From 760948e36b6c89ae7aa62d35d1d8eead952b32c3 Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:13 +0200 Subject: [PATCH 08/10] Refresh benchmark tables Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- README.md | 48 ++++++------- benchmarks/udsv/results.json | 130 +++++++++++++++++------------------ 2 files changed, 89 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 4b94b43..2940e61 100644 --- a/README.md +++ b/README.md @@ -403,31 +403,31 @@ disagreed with uDSV's by more than the harness tolerates is listed with that err | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | ----------------- | -------------------------------------------- | -------------------------------------------- | -| csv-simple-parser | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 390 | ░░░░░░░░░░░░ 58 | -| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 370 | ░░░░░░░░░░░░░░░░░ 85 | -| spliterator | ░░░░░░░░░░░░░░░░░░░░░ 209 | ░░░░░░░░░░░░░░░░░░░░░░░░░ 122 | -| but-csv | ░░░░░░░░░░░░░░░░░░░ 187 | ░░░░░░░░░░░░░░░░░░░░░░░ 112 | -| d3-dsv | ░░░░░░░░░░ 96.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | -| PapaParse | ░░░░░░░░ 74.2 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 192 | +| csv-simple-parser | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 394 | ░░░░░░░░░░░░ 58 | +| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 372 | ░░░░░░░░░░░░ 58 | +| spliterator | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 288 | ░░░░░░░░░░░░░░░░░░░░ 96 | +| but-csv | ░░░░░░░░░░░░░░░░░░░ 183 | ░░░░░░░░░░░░░░░░░░░░░░░ 112 | +| d3-dsv | ░░░░░░░░░░ 93.8 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | +| PapaParse | ░░░░░░░░ 75 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 192 | **litmus_quoted.csv (1.8 MiB, 10K rows), streamed from file** | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | --------------------------- | -------------------------------------------- | -------------------------------------------- | -| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 261 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 153 | -| uDSV (stream) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 229 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 198 | -| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░ 168 | ░░░░░░░░░░░░░░░░░░░ 93 | -| spliterator (stream) | ░░░░░░░░░░░░░░░░░░░░░░░ 153 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 179 | -| PapaParse (stream, count) | ░░░░░░░░░░░░░░ 89.4 | ░░░░░░░░░░░░░░ 71 | -| PapaParse (stream) | ░░░░░░░░░░ 66.8 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 189 | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 247 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 155 | +| uDSV (stream) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 226 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | +| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 197 | ░░░░░░░░░░░░░░░░░░░ 92 | +| spliterator (stream) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 176 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 182 | +| PapaParse (stream, count) | ░░░░░░░░░░░░░░ 84.8 | ░░░░░░░░░░░░░░ 71 | +| PapaParse (stream) | ░░░░░░░░░░░ 66.5 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 190 | **openpayments_covered-recipient-profile_20260603.csv (386 MiB, 1.7M rows), in-memory string** | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | ----------------- | --------------------------------------------------- | ---------------------------------------------- | -| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 142 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.81K | -| d3-dsv | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 101 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.85K | -| spliterator | ░░░░░░░░░░░░░░░░░░░░░ 72.9 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.01K | +| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 143 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.82K | +| d3-dsv | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 103 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.86K | +| spliterator | ░░░░░░░░░░░░░░░░░░░░░░░░ 84.6 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.94K | | PapaParse | Wrong row count! Expected: 1697026, Actual: 1697027 | | | csv-simple-parser | Wrong row count! Expected: 1697026, Actual: 1697027 | | | but-csv | Wrong row count! Expected: 1697026, Actual: 1697027 | | @@ -436,20 +436,20 @@ disagreed with uDSV's by more than the harness tolerates is listed with that err | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | --------------------------- | -------------------------------------------- | ---------------------------------------------- | -| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 307 | ░░ 147 | -| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░ 205 | ░ 98 | -| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░ 184 | ░ 81 | -| spliterator (stream) | ░░░░░░░░░░░░░ 99 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.37K | -| PapaParse (stream) | ░░░░░░░░░░░ 86.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.71K | -| uDSV (stream) | ░░░░░░░░░░ 75.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.67K | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 314 | ░ 137 | +| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 285 | ░ 92 | +| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░ 191 | ░ 82 | +| spliterator (stream) | ░░░░░░░░░░░░░░ 109 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.37K | +| PapaParse (stream) | ░░░░░░░░░░░ 88 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.71K | +| uDSV (stream) | ░░░░░░░░░ 69.9 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.61K | **gb-tuples.csv (1.42 GiB), streamed from file** | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | --------------------------- | -------------------------------------------- | -------------------------------------------- | -| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 269 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 132 | -| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░ 137 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 101 | -| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░ 118 | ░░░░░░░░░░░░░░░░░░░░░░░ 76 | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 277 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 130 | +| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░ 169 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 97 | +| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░░ 122 | ░░░░░░░░░░░░░░░░░░░░░░░ 74 | CSV into arrays of strings is parse-bound, and on that work spliterator is mid-pack: uDSV generates a parser per schema and slices one decoded string, and nothing byte-oriented will catch that. What this harness cannot show is diff --git a/benchmarks/udsv/results.json b/benchmarks/udsv/results.json index bbf57dd..9166a52 100644 --- a/benchmarks/udsv/results.json +++ b/benchmarks/udsv/results.json @@ -3,9 +3,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "spliterator", - "gmean_ms": 8.8, - "mibs": 209.1, - "rss_mib": 122, + "gmean_ms": 6.4, + "mibs": 287.5, + "rss_mib": 96, "rows": 10000, "error": null }, @@ -14,8 +14,8 @@ "mib": 1.8, "name": "uDSV", "gmean_ms": 5, - "mibs": 369.6, - "rss_mib": 85, + "mibs": 371.9, + "rss_mib": 58, "rows": 9999, "error": null }, @@ -23,8 +23,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "PapaParse", - "gmean_ms": 24.9, - "mibs": 74.2, + "gmean_ms": 24.6, + "mibs": 75, "rss_mib": 192, "rows": 10000, "error": null @@ -34,7 +34,7 @@ "mib": 1.8, "name": "csv-simple-parser", "gmean_ms": 4.7, - "mibs": 390.3, + "mibs": 393.6, "rss_mib": 58, "rows": 10000, "error": null @@ -43,8 +43,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "d3-dsv", - "gmean_ms": 19.2, - "mibs": 96.1, + "gmean_ms": 19.7, + "mibs": 93.8, "rss_mib": 196, "rows": 10000, "error": null @@ -53,8 +53,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "but-csv", - "gmean_ms": 9.9, - "mibs": 187.3, + "gmean_ms": 10.1, + "mibs": 183.3, "rss_mib": 112, "rows": 10000, "error": null @@ -63,9 +63,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "spliterator (stream)", - "gmean_ms": 12.1, - "mibs": 152.9, - "rss_mib": 179, + "gmean_ms": 10.5, + "mibs": 175.8, + "rss_mib": 182, "rows": 10000, "error": null }, @@ -73,9 +73,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "uDSV (stream)", - "gmean_ms": 8.1, - "mibs": 228.8, - "rss_mib": 198, + "gmean_ms": 8.2, + "mibs": 226, + "rss_mib": 196, "rows": 9999, "error": null }, @@ -83,9 +83,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "PapaParse (stream)", - "gmean_ms": 27.6, - "mibs": 66.8, - "rss_mib": 189, + "gmean_ms": 27.8, + "mibs": 66.5, + "rss_mib": 190, "rows": 10000, "error": null }, @@ -93,9 +93,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "spliterator (stream, count)", - "gmean_ms": 11, - "mibs": 168.4, - "rss_mib": 93, + "gmean_ms": 9.4, + "mibs": 197.2, + "rss_mib": 92, "rows": 2, "error": null }, @@ -103,9 +103,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "uDSV (stream, count)", - "gmean_ms": 7.1, - "mibs": 261.3, - "rss_mib": 153, + "gmean_ms": 7.5, + "mibs": 247.4, + "rss_mib": 155, "rows": 2, "error": null }, @@ -113,8 +113,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "PapaParse (stream, count)", - "gmean_ms": 20.7, - "mibs": 89.4, + "gmean_ms": 21.8, + "mibs": 84.8, "rss_mib": 71, "rows": 2, "error": null @@ -123,9 +123,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "spliterator", - "gmean_ms": 5292.5, - "mibs": 72.9, - "rss_mib": 3005, + "gmean_ms": 4561.7, + "mibs": 84.6, + "rss_mib": 2942, "rows": 1697026, "error": null }, @@ -133,9 +133,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "uDSV", - "gmean_ms": 2726, - "mibs": 141.6, - "rss_mib": 3814, + "gmean_ms": 2698.3, + "mibs": 143.1, + "rss_mib": 3817, "rows": 1697025, "error": null }, @@ -163,9 +163,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "d3-dsv", - "gmean_ms": 3814.3, - "mibs": 101.2, - "rss_mib": 2852, + "gmean_ms": 3732.1, + "mibs": 103.4, + "rss_mib": 2856, "rows": 1697026, "error": null }, @@ -183,9 +183,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "spliterator (stream)", - "gmean_ms": 3901.6, - "mibs": 99, - "rss_mib": 3373, + "gmean_ms": 3551.6, + "mibs": 108.7, + "rss_mib": 3366, "rows": 1697026, "error": null }, @@ -193,9 +193,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "uDSV (stream)", - "gmean_ms": 5137.7, - "mibs": 75.1, - "rss_mib": 3666, + "gmean_ms": 5524, + "mibs": 69.9, + "rss_mib": 3610, "rows": 1697025, "error": null }, @@ -203,9 +203,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "PapaParse (stream)", - "gmean_ms": 4484.2, - "mibs": 86.1, - "rss_mib": 3708, + "gmean_ms": 4389.3, + "mibs": 88, + "rss_mib": 3710, "rows": 1697026, "error": null }, @@ -213,9 +213,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "spliterator (stream, count)", - "gmean_ms": 1885.5, - "mibs": 204.8, - "rss_mib": 98, + "gmean_ms": 1352.7, + "mibs": 285.4, + "rss_mib": 92, "rows": 2, "error": null }, @@ -223,9 +223,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "uDSV (stream, count)", - "gmean_ms": 1256.2, - "mibs": 307.3, - "rss_mib": 147, + "gmean_ms": 1230.6, + "mibs": 313.7, + "rss_mib": 137, "rows": 2, "error": null }, @@ -233,9 +233,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "PapaParse (stream, count)", - "gmean_ms": 2103.5, - "mibs": 183.5, - "rss_mib": 81, + "gmean_ms": 2026.4, + "mibs": 190.5, + "rss_mib": 82, "rows": 2, "error": null }, @@ -243,9 +243,9 @@ "data": "gb-tuples.csv", "mib": 1454.2, "name": "spliterator (stream, count)", - "gmean_ms": 10587.1, - "mibs": 137.4, - "rss_mib": 101, + "gmean_ms": 8620.6, + "mibs": 168.7, + "rss_mib": 97, "rows": 2, "error": null }, @@ -253,9 +253,9 @@ "data": "gb-tuples.csv", "mib": 1454.2, "name": "uDSV (stream, count)", - "gmean_ms": 5413.6, - "mibs": 268.6, - "rss_mib": 132, + "gmean_ms": 5258, + "mibs": 276.6, + "rss_mib": 130, "rows": 2, "error": null }, @@ -263,9 +263,9 @@ "data": "gb-tuples.csv", "mib": 1454.2, "name": "PapaParse (stream, count)", - "gmean_ms": 12378.5, - "mibs": 117.5, - "rss_mib": 76, + "gmean_ms": 11943.9, + "mibs": 121.7, + "rss_mib": 74, "rows": 2, "error": null } From 049938d92b89ebce8080f8c2af60f2f9e94d0caa Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:58:21 +0200 Subject: [PATCH 09/10] Split string sources as text in TextSpliterator.from and JSONSpliterator.from A string is already text, so encoding it, scanning the bytes and decoding each row back was pure overhead: 7.8us per call on a 40-line string against 1us for split. textRows splits the string on the delimiter text with the engine's crlf, skipEmpty, drop and take semantics, 2.5us per call. Gated by canSplitAsText: no byte position, no quote handling, UTF-8 only. The streaming window path shares the splitter. Parity with the byte route is pinned across delimiters and options. Claude-Session: https://claude.ai/code/session_01MonKZmMeL1Jvo7i6EHmbYx --- AGENTS.md | 1 + lib/formats/JSONSpliterator.ts | 31 +++++++++- lib/formats/TextSpliterator.ts | 24 +++++++- lib/io/windowed-text.ts | 109 +++++++++++++++++++++++++-------- test/formats/text-rows.test.ts | 79 ++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 30 deletions(-) create mode 100644 test/formats/text-rows.test.ts diff --git a/AGENTS.md b/AGENTS.md index 88d0661..a10b44e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,6 +125,7 @@ Boyer-Moore-Horspool fallback, and ~8–17× for `searchAll`. list parser. `count`/`countAsync` test raw slices for ASCII whitespace to stay in agreement without decoding. Pass `trim: false` for byte parity. - **`JSONSpliterator`** — Wraps `TextSpliterator`-style logic, additionally calls `JSON.parse` on each line. +- **A string source to `TextSpliterator.from` or `JSONSpliterator.from` is split as text** (`textRows` in `lib/io/windowed-text.ts`): no encode, no byte scan, no per-row decode. Gated by `canSplitAsText`: no byte `position`, no quote handling, no encoding other than UTF-8. A 40-line string went 7.8µs → 2.5µs per call against 1µs for `split`+`trim`; the remainder is the `Sequence` and generator. This is the shape of the 80-odd stdout-splitting call sites in mailwoman. `test/formats/text-rows.test.ts` pins parity with the byte route across delimiters, `crlf`, `skipEmpty`, `drop`/`take`. - **`CSVSpliterator`** — Two-level splitting: first splits rows (newline), then splits each row into columns (comma). Supports `mode: "array" | "object" | "entries"`, header normalization, and per-column transformers. **`trim` is on by default** and trims every column after diff --git a/lib/formats/JSONSpliterator.ts b/lib/formats/JSONSpliterator.ts index 709f995..8c0b7bd 100644 --- a/lib/formats/JSONSpliterator.ts +++ b/lib/formats/JSONSpliterator.ts @@ -10,7 +10,7 @@ import { Spliterator, type SpliteratorInit } from "#core/Spliterator" import { type CommentInput, createCommentFilter, createTextCommentFilter } from "#formats/comment-filter" import type { AsyncDataResource } from "#internal/shared" import { type AdaptiveSourceInit, openDelimitedRows } from "#io/adaptive-source" -import { windowedTextRows } from "#io/windowed-text" +import { canSplitAsText, textRows, windowedTextRows } from "#io/windowed-text" import { AsyncSequence } from "#iterators/AsyncSequence" import { Sequence } from "#iterators/Sequence" @@ -36,9 +36,36 @@ function* parseRows( source: CharacterSequenceInput, { comment, ...options }: SpliteratorInit & JSONSpliteratorInit = {} ): Generator { + let rowCursor = 0 + + // A string is already text: split it as text rather than encoding, scanning and decoding each row back. + if (typeof source === "string" && canSplitAsText(options)) { + const parseableText = createTextCommentFilter(comment) + + for (const row of textRows(source, options)) { + if (parseableText && !parseableText(row)) continue + + let parsed: T + + try { + parsed = JSON.parse(row) as T + } catch (parsedError) { + const error = new SyntaxError(`Failed to parse JSON at row ${rowCursor}`) + error.cause = parsedError + + throw error + } + + yield parsed + + rowCursor++ + } + + return + } + const decoder = new TextDecoder() const parseable = createCommentFilter(comment) - let rowCursor = 0 const spliterator = Spliterator.fromSync(source, options) diff --git a/lib/formats/TextSpliterator.ts b/lib/formats/TextSpliterator.ts index ed5585a..4f9ba64 100644 --- a/lib/formats/TextSpliterator.ts +++ b/lib/formats/TextSpliterator.ts @@ -9,7 +9,7 @@ import { CharacterSequence, type CharacterSequenceInput } from "#core/CharacterS import { Spliterator, type SpliteratorInit } from "#core/Spliterator" import type { AsyncDataResource } from "#internal/shared" import { type AdaptiveSourceInit, openDelimitedRows } from "#io/adaptive-source" -import { windowedTextRows } from "#io/windowed-text" +import { canSplitAsText, textRows, windowedTextRows } from "#io/windowed-text" import { AsyncSequence } from "#iterators/AsyncSequence" import { Sequence } from "#iterators/Sequence" @@ -65,8 +65,28 @@ function* decodeRows( source: CharacterSequenceInput, { encoding, fatal, ignoreBOM, trim = true, ...options }: TextSpliteratorInit & SpliteratorInit = {} ): Generator { - const decoder = new TextDecoder(encoding, { fatal, ignoreBOM }) const dropBlank = trim && (options.skipEmpty ?? true) + + // A string is already text: split it as text instead of encoding it, scanning the bytes and decoding each row back. + // The byte route cost 7.8µs for a 40-line string against 1µs for `split`; this is what the stdout-splitting callers + // pay per call. + if (typeof source === "string" && canSplitAsText({ ...options, encoding })) { + for (const row of textRows(source, options)) { + if (trim) { + const trimmed = row.trim() + + if (dropBlank && !trimmed) continue + + yield trimmed + } else { + yield row + } + } + + return + } + + const decoder = new TextDecoder(encoding, { fatal, ignoreBOM }) let rowCursor = 0 const spliterator = Spliterator.fromSync(source, options) diff --git a/lib/io/windowed-text.ts b/lib/io/windowed-text.ts index c0f96eb..821aef0 100644 --- a/lib/io/windowed-text.ts +++ b/lib/io/windowed-text.ts @@ -5,10 +5,87 @@ */ import type { AsyncSpliterator } from "#core/AsyncSpliterator" -import { type CharacterSequence, Delimiters } from "#core/CharacterSequence" -import { Spliterator } from "#core/Spliterator" +import { CharacterSequence, Delimiters } from "#core/CharacterSequence" +import { Spliterator, type SpliteratorInit } from "#core/Spliterator" import { batched, type BatchedAsyncIterable } from "#iterators/AsyncSequence" +/** + * Split `text` on `delimiter` into `rows`, the way the byte engine would have cut the encoded bytes: a carriage return + * before a delimiter goes with the delimiter when `crlf` is on, one at the end of the text is data, and the piece after + * the last delimiter is a row even when empty unless `skipEmpty` drops it, matching `String.prototype.split`. + * + * Returns the number of rows pushed. Shared by the streaming window path and the string fast path of the text and JSON + * spliterators. + */ +export function splitTextRows( + text: string, + delimiter: string, + crlf: boolean, + skipEmpty: boolean, + rows: string[], + limit = Infinity +): number { + const delimiterLength = delimiter.length + const before = rows.length + let position = 0 + + while (rows.length - before < limit) { + const found = text.indexOf(delimiter, position) + let end = found === -1 ? text.length : found + + if (crlf && found !== -1 && end > position && text.charCodeAt(end - 1) === Delimiters.CarriageReturn) { + end-- + } + + if (!(skipEmpty && end === position)) { + rows.push(text.slice(position, end)) + } + + if (found === -1) break + + position = found + delimiterLength + } + + return rows.length - before +} + +/** + * Whether a string source can be split as text rather than encoded, scanned as bytes and decoded per row: no byte + * `position` to honour, no quote handling, and no encoding other than UTF-8 asked for (a string is already text). + */ +export function canSplitAsText(init: SpliteratorInit & { encoding?: string }): boolean { + return !init.position && !init.enableQuoteHandling && (init.encoding === undefined || /^utf-?8$/i.test(init.encoding)) +} + +const sharedDecoder = new TextDecoder() + +/** + * The delimiter as text. `delimiter` is whatever the caller gave the engine: a string, bytes, or nothing for a newline. + */ +export function delimiterText(delimiter: SpliteratorInit["delimiter"]): string { + if (delimiter === undefined) return "\n" + + return typeof delimiter === "string" ? delimiter : sharedDecoder.decode(new CharacterSequence(delimiter)) +} + +/** + * Every row of a string source, split as text. Measured against the byte route (encode, scan, decode per row) on a + * 40-line string: 7.8µs → see `TextSpliterator.from`. The whole string is split before the first row is yielded, which + * for the in-memory strings this serves (a command's output, a file read whole) is the cheap part. `drop` and `take` + * count after `skipEmpty`, as the engine's do. + */ +export function* textRows(text: string, init: SpliteratorInit): Generator { + const rows: string[] = [] + const drop = Math.max(0, init.drop ?? 0) + const take = Math.max(0, init.take ?? Infinity) + + splitTextRows(text, delimiterText(init.delimiter), init.crlf ?? false, init.skipEmpty ?? true, rows, drop + take) + + for (let index = drop; index < rows.length; index++) { + yield rows[index]! + } +} + export interface WindowedTextInit { /** * The delimiter as the engine was given it. Decoded once with `decoder` so the window's text splits on the same @@ -47,8 +124,7 @@ export function windowedTextRows(engine: AsyncSpliterator, init: WindowedTextIni async function* windows(engine: AsyncSpliterator, init: WindowedTextInit): AsyncGenerator { const { crlf, skipEmpty, decoder } = init - const delimiterText = decoder.decode(init.delimiter) - const delimiterLength = delimiterText.length + const delimiter = decoder.decode(init.delimiter) let rowIndex = 0 try { @@ -82,30 +158,11 @@ async function* windows(engine: AsyncSpliterator, init: WindowedTextInit): Async } const rows: string[] = [] - let position = 0 // The window holds whole records with the delimiters between them and none after the last, exactly as the - // engine cut them: the piece after the last delimiter is the last record, not a partial one. - while (true) { - const found = text.indexOf(delimiterText, position) - let end = found === -1 ? text.length : found - - // A carriage return before a delimiter is part of the delimiter; one at the end of the input is data, and the - // engine already trimmed the one before the last delimiter it cut. - if (crlf && found !== -1 && end > position && text.charCodeAt(end - 1) === Delimiters.CarriageReturn) { - end-- - } - - if (!(skipEmpty && end === position)) { - rows.push(text.slice(position, end)) - } - - if (found === -1) break - - position = found + delimiterLength - } - - rowIndex += rows.length + // engine cut them: the piece after the last delimiter is the last record, not a partial one, and the engine + // already trimmed the carriage return before the last delimiter it cut. + rowIndex += splitTextRows(text, delimiter, crlf, skipEmpty, rows) if (rows.length) { yield rows diff --git a/test/formats/text-rows.test.ts b/test/formats/text-rows.test.ts new file mode 100644 index 0000000..73b6b41 --- /dev/null +++ b/test/formats/text-rows.test.ts @@ -0,0 +1,79 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + */ + +import { JSONSpliterator, TextSpliterator } from "spliterator" +import { describe, expect, test } from "vitest" + +/** + * `TextSpliterator.from` and `JSONSpliterator.from` split a string source as text; a byte source goes through the + * engine. Every case here runs both and expects the same rows. + */ +const encoder = new TextEncoder() + +const texts: Array<[string, string]> = [ + ["plain", "one\ntwo\nthree\n"], + ["no trailing newline", "one\ntwo\nthree"], + ["blank rows", "one\n\n \ntwo\n\n"], + ["crlf", "one\r\ntwo\r\nthree\r\n"], + ["lone cr at end of input", "one\ntwo\r"], + ["multi-byte", "é한😀 one\nzwei 한국어\n三\n"], + ["empty", ""], + ["only newlines", "\n\n\n"], + ["single row", "x"], +] + +const inits: Array<[string, Parameters[1]]> = [ + ["defaults", {}], + ["trim off, empties kept", { trim: false, skipEmpty: false }], + ["crlf", { trim: false, crlf: true }], + ["crlf, empties kept", { trim: false, crlf: true, skipEmpty: false }], + ["drop and take", { drop: 1, take: 2 }], + ["take zero", { take: 0 }], + ["multi-character delimiter", { delimiter: "e\n", trim: false }], + ["byte delimiter", { delimiter: encoder.encode("\n"), trim: false }], +] + +describe("string sources split as text", () => { + for (const [label, text] of texts) { + for (const [initLabel, init] of inits) { + test(`${label}, ${initLabel}`, () => { + expect(TextSpliterator.from(text, init).toArray()).toEqual( + TextSpliterator.from(encoder.encode(text), init).toArray() + ) + }) + } + } + + test("a byte position or quote handling takes the byte route", () => { + const text = 'é,"a,b",c' + + expect(TextSpliterator.from(text, { delimiter: ",", position: 3, trim: false }).toArray()).toEqual( + TextSpliterator.from(encoder.encode(text), { delimiter: ",", position: 3, trim: false }).toArray() + ) + + expect(TextSpliterator.from(text, { delimiter: ",", enableQuoteHandling: true }).toArray()).toEqual([ + "é", + '"a,b"', + "c", + ]) + }) + + test("JSON rows, comments and errors match the byte route", () => { + const text = '// header\n{"a":1}\n \n{"b":"한"}\n' + const init = { comment: "//" } + + expect(JSONSpliterator.from(text, init).toArray()).toEqual( + JSONSpliterator.from(encoder.encode(text), init).toArray() + ) + + expect(JSONSpliterator.from(text, init).toArray()).toEqual([{ a: 1 }, { b: "한" }]) + expect(() => JSONSpliterator.from('{"a":1}\nnope\n').toArray()).toThrow("Failed to parse JSON at row 1") + + expect(() => JSONSpliterator.from(encoder.encode('{"a":1}\nnope\n')).toArray()).toThrow( + "Failed to parse JSON at row 1" + ) + }) +}) From fea05e409a11fdcbb99fe8ab41839d3f695792e6 Mon Sep 17 00:00:00 2001 From: Teffen Ellis <592134+GirlBossRush@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:46:42 +0200 Subject: [PATCH 10/10] Add scan-only and worker-parallel row-count rows to the uDSV benchmark The count adapters full-parse because uDSV's do; the library's scan-only path was never shown. Two new adapters, labeled as their own rows so the full-parse counts stay for apples-to-apples: - spliterator (stream, row count): CSVSpliterator.countAsync, counting engine rows without decoding columns. - spliterator (stream, row count, N workers): AsyncSpliterator.segments plus one persistent worker per core, each counting rows in its own range with countAsync's semantics and posting a single number back. Verified exact agreement with the sequential parser and expected.json on all three datasets. Segments are quote-blind, so embedded newlines in quoted fields would miscount in parallel; demonstrated on a synthetic file and documented in the adapter, README, and HANDOFF. 715/1091/393 MiB/s sequential and 669/2015/721 parallel against uDSV's 249/310/271 on litmus, openpayments, and gb-tuples. README tables and results.json regenerated from one fresh sweep; HANDOFF.md section 3.14 covers the work. --- HANDOFF.md | 578 ++++++++++++++++++ HANDOFF_SUGGESTIONS.md | 32 + README.md | 79 +-- .../udsv/adapters/parallel-count-worker.ts | 62 ++ .../adapters/spliterator-parallel-count.ts | 90 +++ .../udsv/adapters/spliterator-rows-count.ts | 26 + benchmarks/udsv/results.json | 190 ++++-- benchmarks/udsv/sweep.ts | 8 +- 8 files changed, 965 insertions(+), 100 deletions(-) create mode 100644 HANDOFF.md create mode 100644 HANDOFF_SUGGESTIONS.md create mode 100644 benchmarks/udsv/adapters/parallel-count-worker.ts create mode 100644 benchmarks/udsv/adapters/spliterator-parallel-count.ts create mode 100644 benchmarks/udsv/adapters/spliterator-rows-count.ts diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 0000000..92a2b38 --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,578 @@ +# Handoff: spliterator CSV performance work, 2026-10-08/09 + +Sections 3.1–3.13 were committed after this document was first written (the branch was rebased onto the 9.1.0 +release commit, see 3.12). Section 3.14 (row-count benchmark rows) is uncommitted on `main`, as are this file and +`HANDOFF_SUGGESTIONS.md`. `yarn compile && yarn test --run && yarn lint` is green (835 tests). The person owning +this repo decides what lands. + +This document is for an agent arriving cold. It covers what was measured, what was changed, why, +what was tried and rejected, and where the remaining headroom is thought to be. Read `AGENTS.md` +first for the architecture; this file assumes it. + +--- + +## 1. Where this started + +The goal was to benchmark spliterator's CSV parsing against [uDSV](https://github.com/leeoniya/uDSV) +using uDSV's own bench harness (checked out at `../uDSV`), on realistic large files from `/mnt/mw`, +then chase whatever the benchmark exposed, and publish an honest benchmark section in `README.md` +reproducible from a script in the repo. + +Datasets used: + +| File | Size | Shape | +| ------------------------------------------------------------------------------------ | ------- | ---------------------------------------------------- | +| `../uDSV/bench/data/litmus_quoted.csv` | 1.9 MB | uDSV's synthetic quoted fixture (`litmus_gen.cjs`) | +| `/mnt/mw/record-matcher/sources/openpayments_covered-recipient-profile_20260603.csv` | 386 MB | wide, quoted, real | +| `/mnt/mw/ppd/2026-07-22/gb-tuples.csv` | 1.45 GB | quoted, exceeds V8's string limit so it only streams | + +Scratch copies for micro-timing (not in the repo, in the session scratchpad +`/tmp/claude-1000/-home-lab-Projects-spliterator/e5371494-bd4b-49fc-a5b1-0e919d7c66d6/scratchpad/`): +`gb300.csv` (first 300 MB of gb-tuples, quoted), `gb300-noq.csv` (same rows with quotes stripped, +266 MB), `count.mjs` (streams a file with `bulkThreshold: 0`, `mode: "array"`, `header: false`, +`trim: false`, counts rows, prints ms and MiB/s). + +--- + +## 2. The benchmark harness integration (`benchmarks/udsv/`) + +- `sweep.ts` spawns uDSV's `bench/runone.cjs` per parser per dataset, unmodified, and collects its + JSON. The harness's metric is a geometric mean over 3-second cycles, reported as MiB/s. +- The harness reads the whole file into a string unless the adapter path contains + `/streaming/non-retained`. A 1.45 GB file cannot be a string (`ERR_STRING_TOO_LONG`), so the + sweep writes `runone-streaming.generated.cjs`, a copy with that test widened to also match + `-count.js$`, and uses it for the count adapters. +- uDSV drops the header row; the harness verifies row counts with a tolerance of 2 below + `expected.json`. For a dataset with no entry the sweep computes one with uDSV itself and adds 1, + so both uDSV and header-keeping parsers pass. A file too large for a string gets a placeholder + entry and runs unverified. +- Memory is not taken from the harness (its figure is the largest RSS step _between_ cycles, which + reads zero for a parser that allocates on the first parse). Each run goes through + `/usr/bin/time -f %M`; peak RSS minus the baseline the runner reports before loading the parser + is recorded as "Peak RSS above baseline". +- Adapters in `benchmarks/udsv/adapters/`: `spliterator-memory` (string in), `spliterator-stream` + (file in, rows retained), `spliterator-count`, `udsv-count`, `papaparse-count` (file in, rows + discarded). Count adapters locate uDSV through `UDSV_ROOT`. +- `render.ts` turns `results.json` into Markdown tables (the user explicitly wanted Markdown, not + box-drawn code blocks, and a memory column). `README.md` has a `## Benchmarks` section built from + it. `test/benchmarks/udsv-render.test.ts` covers the renderer. + +Latest harness numbers (2026-10-09, after 3.7; background load ~3, equal for every parser): + +| Dataset / mode | spliterator | uDSV | PapaParse | +| ----------------------------------------------- | ----------- | ---- | -------------- | +| litmus_quoted 1.9 MB, in-memory string → arrays | 209 MiB/s | 370 | 74 | +| litmus_quoted, stream, rows retained | 153 | 229 | 67 | +| litmus_quoted, stream+count | 168 | 261 | 89 | +| openpayments 386 MB, in-memory string → arrays | 73 | 142 | (fails verify) | +| openpayments, stream, rows retained | 99 | 75 | 86 | +| openpayments, stream+count | 205 | 307 | 184 | +| gb-tuples 1.45 GB, stream+count | 137 | 269 | 118 | + +The in-memory openpayments figure is one sample per 3-second cycle and includes a major GC of the +previous 1.3 GB result; it moved 78 → 73 between two sweeps with no code change in between, so +treat ±10% there as noise. Standalone, a single parse of that string is 3.2 s (was 3.8 s). + +Before this work spliterator's streaming count was 88 and 148 MiB/s on those two files. The full +tables with memory columns are in `README.md` and `benchmarks/udsv/results.json`. + +--- + +## 3. Changes made, in order of measured payoff + +### 3.1 `AsyncSequence.next()` awaited its opener on every pull (biggest win, not planned) + +`lib/iterators/AsyncSequence.ts`. The pull loop did `await this.#openUpstream()` unconditionally. +After the first pull that is an extra async frame per item on every streamed format. Fixed by +reading `this.#upstream ?? this.#syncUpstream` first and only awaiting the opener when both are +null. + +Measured on the 300 MB quoted stream: 2.9 s → 2.5 s. `TextSpliterator` over the same file +2.0 s → 1.6 s. 20–40% on streamed rows. This was found by profiling, not by the plan. + +### 3.2 Batched sources (`BATCHED_SOURCE`, `batched()`, `BatchedAsyncIterable`) + +Same file. An upstream can be marked as yielding arrays; `next()` then walks each array in place +with no `await` per element, skipping empty arrays. Needed because the windowed cell scan (3.3) +was _slower_ than the per-row path until rows crossed the sequence boundary in bulk: rows yielded +singly from an async generator 4.8 s, per-row path 3.2 s, batched 2.9 s. A batched source that has +its header consumed must put the rest of that batch back in front: `reheadBatches` in +`lib/formats/csv-cells.ts`. Tests in `test/iterators/batched.test.ts`. + +### 3.3 Windowed cell scanning on the streaming CSV path + +- `AsyncSpliterator.nextWindow()` (`lib/core/AsyncSpliterator.ts`): fills as `next()` does, then + dequeues _every_ queued range at once and returns the contiguous view from the first range's + start to the last range's end. Whole records only; about a high-water mark (64 KiB) of rows per + await. The engine's `skipEmpty`/`drop`/`take` do **not** apply to windows; the CSV layer applies + them itself. Invariant tested in `test/core/nextWindow.test.ts`: + `windows.join(delimiter) === source`, with a trailing empty window when the source ends on a + delimiter. +- `scanCsvCellsStreaming` (`lib/formats/csv-cells.ts`): per window, decode with the fatal decoder + and run `scanCsvCells` (the existing in-memory kernel driver) over it. Nothing carries across + windows because the engine cut the records with quote state in hand. A window that is not valid + UTF-8 takes the per-row path for that window alone (the `fallback` callback). Output is a batched + source. +- `CSVSpliterator.fromAsync` (`lib/formats/CSVSpliterator.ts`): `openRows` now awaits + `CharacterSequence.whenReady()` on the streaming branch too, because `scanWindows` checks + `hasScanner()` and a fresh process otherwise took the row path for its first file. **This was + invisible in the test suite** (the scanner was already loaded by earlier tests) and only showed + up in a standalone run. Header handling: if the opened rows are batched, the first non-empty + batch's first row is the header and the remainder is re-headed. +- `columnScan: "rows"` opts out and remains the parity reference. Tests in + `test/formats/CSVSpliterator.test.ts` under `describe("streaming cell scan")`: parity at + `highWaterMark: 32`, a quoted field longer than a window, an invalid-UTF-8 window, `skipEmpty: + false`, early exit closes the source. + +### 3.4 A string source is the decoded text + +`CSVSpliterator.from(string)` passes the string itself to the cell scanner instead of decoding the +bytes it had encoded from it. Offsets agree because a lone surrogate encodes as U+FFFD, one UTF-16 +unit either way (the surrogate survives where the row path yields U+FFFD; tested in +`test/formats/csv-cells.test.ts`). On the 386 MB string: 4.2 s → 3.8 s and peak memory +2237 → 1465 MiB (the second copy of the source is gone). Side effect: `columnScan: "rows"` no longer +saves memory on a string source, so that README claim was removed and the reason recorded in +`AGENTS.md`. + +### 3.5 Unquoting + +`lib/formats/csv-columns.ts`. + +- `unquoteColumn(value, escaped?)`: only calls `replaceAll('""', '"')` when something says a + doubled quote exists. If `escaped` is undefined it runs one `indexOf('""')`; the two-pass form + with unconditional `replaceAll` was 17% of a streamed quoted parse. +- `splitQuotedString` (row path) is now a fused split+unquote walk with a `sawDoubledQuote` flag + and `unquoteSlice(line, start, end, sawDoubledQuote)`, so a quoted cell is sliced once. Edge cases + in `test/core/QuoteHandling.test.ts`. + +### 3.6 Kernel escape flag (last change, 2026-10-09) + +`wasm/src/lib.rs` `scan_csv_cells`. New `CELL_FLAG_HAS_ESCAPE = 4`, set when a quote _opens_ +directly after a quote byte (the second half of `""` inside a quoted field). Reasoning on why that +rule is exact: `""` as an empty quoted cell is open-then-close, so no flag; `"a""b"` is +open-close-open(prev is quote → flag)-close; `x""y` with quotes not at the edges is open-close, no +flag, and `unquoteColumn` would not have stripped it anyway. The rule marks every pair where the +first quote closed a field, which is exactly what the row path's `sawDoubledQuote` captures. + +- The carried state slot 4 of the results header is now a flags word + (`HAS_QUOTE | HAS_ESCAPE`), not a boolean. Renamed `cellHasQuote` → `cellFlags` in + `lib/core/wasm_module.ts`, `lib/core/CharacterSequence.ts` (`CellScanState`, + `WasmCellScanResult`), `lib/formats/csv-cells.ts`, `test/core/wasm.test.ts`. +- Across a window edge the first quote of a pair can be the last byte of the previous window; the + kernel reads `previous_byte` (already passed for CR handling) when `offset == 0`. Tested. +- `csv-cells.ts` reads the flag itself and no longer calls `unquoteColumn` at all (see 3.7). +- `wasm/build.sh` was run; `lib/core/wasm_base64.ts` is regenerated (3343 bytes). + +Interleaved A/B on the 300 MB quoted stream (a git worktree with the `indexOf` form, four pairs): + +| Build | Runs | +| ---------------- | ------------------------ | +| `indexOf` search | 2.80, 2.57, 2.61, 2.48 s | +| kernel flag | 2.67, 2.44, 2.40, 2.37 s | + +~5%, every pair in the same direction. + +### 3.7 The in-memory gap was the scavenger, not the parser (2026-10-09) + +The suggestion document (`HANDOFF_SUGGESTIONS.md`) read the 3.5× litmus gap as per-cell work and +proposed a fused `indexOf` loop over the string. Profiling one litmus parse first said otherwise: + +| Measurement on `litmus_quoted.csv` (1.9 MB, 10k rows × 20 quoted cells) | Value | +| ----------------------------------------------------------------------- | ------- | +| encode string → bytes | 0.17 ms | +| kernel boundaries only (`CharacterSequence.scanCells` loop) | 2.6 ms | +| scan + slice, rows discarded, min of 200 | 8.6 ms | +| same, rows **kept**, p50 | 19 ms | +| uDSV `stringArrs`, p50 | 5.0 ms | +| harness-style geometric mean (what `results.json` reports) | 17.6 ms | + +Discarding rows was a steady 9–10 ms per parse; keeping them doubled it, and `--cpu-prof` +attributed the difference to `(garbage collector)`: 43% of the kept run against 7% for a plain +`split`. `--trace-gc-nvp` gave the mechanism: each scavenge saw ~22 MB allocated since the last +one with ~12 MB surviving (the result built so far), 9–11 ms per scavenge, about one per parse. +uDSV allocated ~8.5 MB per parse, scavenged once per ~3.5 parses, and each one survived ~3 MB in +under 1 ms. `%DebugPrint` on the results showed cells with the same representation in both +(sliced one-byte strings at ≥13 chars, sequential copies below), but our row arrays had a +`FixedArray[43]` backing store behind 20 elements where uDSV's had `[20]`. + +Two allocations per cell were the whole difference: + +1. `unquoteColumn` sliced the quoted outer string, then sliced its inside. The outer string was + garbage on every quoted cell. `cell()` now checks the quote bytes in the source text and slices + the inside once; `CELL_FLAG_HAS_ESCAPE` says whether to `replaceAll`. +2. Rows grew by `push` from `[]`: two discarded stores per row and a 43-slot survivor. Rows are + now filled into `template.slice()` where the template is an exact-width array of `""` the + width of the previous row (uDSV's `rowTpl.slice()`), with `row.slice(0, width)` for a + narrower row and plain index assignment past the end for a wider one. + +A trap worth knowing: `row[width++] = cell(...)` evaluates the member expression first, so +`width` was already incremented when `cell()` ran and the BOM strip at `width === 0` never +fired. The BOM parity tests caught it; the value is now assigned to a local first. + +Result on litmus, same machine, same session: + +| Metric | Before | After | +| ------------------------------------ | ------- | ------ | +| harness-style gmean | 17.6 ms | 8.7 ms | +| GC per parse (perf_hooks `gc`) | 6.9 ms | 0.9 ms | +| min | 9.0 ms | 7.1 ms | +| uDSV, same loop | 5.4 ms | 5.1 ms | +| 300 MB quoted stream, rows discarded | 2.4 s | 2.1 s | + +The remaining litmus gap is ~1.7× and is now in the min, not the mean: ~2.6 ms of kernel +round-trip (encode, stage into WASM memory, copy the result block out, rebase) plus the slicing +loop. The fused-string-loop idea from the suggestions would attack the kernel share; it is still a +third implementation to keep in parity, and it is now worth ~2.6 ms of 8.7, not the 3.5× it was +pitched at. + +`unquoteColumn`'s `escaped` parameter added in 3.6 was removed again as dead; the row path has its +own `sawDoubledQuote`, and the cell path no longer calls it. + +### 3.8 Compiled row builders for object and entries mode (2026-10-09) + +Move 1 from `HANDOFF_SUGGESTIONS.md`. Baseline on litmus with `header: true`, harness-style gmean: +array 9.8 ms, object 62.5 ms, entries 38.4 ms. `--cpu-prof` on object mode: `zipSync` 29%, the +`object` emitter body 23%, GC 22%. + +Step 1, replace the `zipSync` iterator (an iterator step and a 3-tuple per cell) with an indexed +loop when both arguments are arrays: object 62.5 → 29.6 ms, entries 38.4 → 20.1 ms. + +Step 2, compile a builder per bound header with `new Function`, a literal +`{ "id": f[0](0 < c.length ? c[0] : m), ... }` with the header names as string-literal keys. A +scratch prototype on the retained array rows measured the loop at 6.7 ms per 10k objects and the +literal at 0.25 ms (results escaping to a global, so not escape analysis). In the library it is +adaptive: the loop handles the first 32 rows of a header, then the builder is compiled once and +cached in a `WeakMap` keyed by the bound header array. Result: object 10.0 ms, entries 17.4 ms, +array 10.0 ms. Object mode is now free relative to array mode; entries still pays one 3-element +array per cell. + +Details worth knowing: + +- `JSON.stringify(key)` makes any header a valid literal key. `"__proto__": v` in a literal invokes + the setter, same as `record["__proto__"] = v` did, so a string is ignored on both paths. +- A `new Function` that throws (CSP without `unsafe-eval`) is recorded as `false` on the header's + state and the loop stays, no retry per row. +- Headers above 2048 columns never compile. +- Columns past the header are appended as `column_` after the literal. +- `bindTransformers` had a latent bug the new tests surfaced: a header named `__proto__` bound + `Object.prototype` as its transformer (`byName[columnName] ?? identity` on a plain object). It now + checks `Object.hasOwn`. +- `test/formats/row-emitters.test.ts` checks every expectation on row 3 (loop) and row 81 + (compiled). + +### 3.9 Stepping scanner (2026-10-09) + +`--cpu-prof` on the 300 MB stream after 3.8: `scanCsvCells` generator 13%, the `windows` async +generator plus `Array.from` 11%, `toColumns` map callback 8%, `AsyncSequence.next` 6%. The cell +loop is now `CellRowScanner.nextBatch(out)`, a stepping object that pushes one kernel batch of rows +(≤ 4096 cells) into the caller's array. The in-memory parser yields a fresh array per batch and +walks it in its own generator (one generator per row, not two); the streaming path collects a whole +window into one array with no flattening copy; `scanCsvCells` stays as the flat generator for tests +and the small-file bulk parser. litmus in-memory gmean 9.7 → 8.3 ms, object 9.0 → 7.8 ms, entries +13.0 → 11.2 ms; the 300 MB quoted stream 2.1 → 1.9 s (150 MiB/s). + +Where the in-memory litmus parse (8.3 ms, uDSV 5.0 ms) goes now, from the profile: JS cell loop +~2.0 ms (10 ns/cell), kernel ~0.95 ms, `scanCells` staging/copy/rebase ~0.35 ms, the format's +generator and `Sequence` ~1 ms, GC ~0.8 ms, encode ~0.3 ms. Nothing left is a single large item. + +The per-row `map` op on the stream (`toColumns` plus the emitter, ~8%) was considered and left: +applying it per batch inside the producer would save one closure call per row but not the +sequence's own per-item step, and the header is bound after the first batch arrives, which +complicates where the emitter can run. + +### 3.10 JSONL and text streaming decode per window (2026-10-09) + +Priorities were re-read from mailwoman's actual usage (`scratchpad/2026/10/09/spliterator.code-search` +in the mailwoman repo): 110 `JSONSpliterator.fromAsync` call sites, ~60 CSV/TSV `fromAsync` over +multi-million-row corpus files in object mode with `normalizeKeys: false`, 82 `TextSpliterator.from` +over small in-memory text (stdout splitting), 5 XLSX. No numeric or columnar consumers. So the +typed-columnar kernel idea (section 9) has no customer and was dropped; JSONL streaming moved to +the top. + +Baseline on a 120 MB synthetic JSONL of 600k rows: `readline` + `JSON.parse` 783 ms, +`JSONSpliterator.fromAsync` 1289 ms, `TextSpliterator.fromAsync` without parsing 281 ms, the raw +engine 138 ms. A prototype that pulled `nextWindow()`, decoded once, split the text on `\n` and +parsed each slice ran 584 ms. The per-row cost was a `TextDecoder.decode` call per row (a few +hundred ns of fixed overhead in Node) plus the per-row `await`. + +`lib/io/windowed-text.ts` is that prototype as a batched source: decode per window, split on the +decoded delimiter (multi-character works), `crlf` and `skipEmpty` applied in the split, rows out as +batches. `TextSpliterator.fromAsync` and `JSONSpliterator.fromAsync` take it when the adaptive +source returns a streaming engine; the small-file bulk branch is unchanged. `drop`/`take` moved +from engine options to sequence ops on these two paths (they still count after `skipEmpty`). A +`fatal` decoder failure on a window falls back to per-row decoding to name the row. A text version +of the comment filter mirrors the byte one. + +| JSONL | before | after | readline + JSON.parse | +| ----------------------------------- | ------- | ------- | --------------------- | +| 120 MB synthetic, 600k rows | 1289 ms | 707 ms | 702 ms | +| 500 MB BAN corpus slice, 1.27M rows | — | 1948 ms | 2092 ms | +| Text rows only, 120 MB | 281 ms | 241 ms | | + +The 120 ms between the library (707) and the prototype (584) is the sequence's per-row op step +and the `map` closure; parsing inside the batch producer would recover some of it at the cost of +moving error attribution out of the sequence. Not done. + +Tests: `test/io/windowed-text.test.ts`, every case at `highWaterMark: 48` with 7-byte chunks so +rows and multi-byte characters straddle windows, checked against the sync `from`. + +### 3.11 Kernel batch as a view (2026-10-09) + +`CharacterSequence.scanCells` returned a copy of the result block rebased to absolute units in a JS +loop, 9% of a streamed parse. It now returns the shared-memory view, window-relative, plus a +`unitBase` the consumer adds. Safe because `CellRowScanner.nextBatch` turns the whole batch into +rows before returning and runs no user code; the AGENTS.md gotcha that forbade removing the copy +was rewritten to say when a copy is required (any new consumer that yields or calls out mid-batch). +386 MB streamed into objects 1869 -> 1685 ms, litmus in-memory 8.3 -> 7.4 ms, 300 MB quoted stream +1.9 -> 1.8 s. + +### 3.12 Consumer CI break on 9.1.0: import.meta.resolve (2026-10-09) + +mailwoman's Docusaurus build failed after bumping to spliterator 9.1.0. Docusaurus loads plugins +through jiti, which transpiles ESM dependencies to CommonJS on the fly: it rewrites +`import.meta.url` and leaves `import.meta.resolve` alone, so Node refused +`out/lib/internal/node-modules.js` with "Cannot use 'import.meta' outside a module". +`workerEntryUrl` now resolves through `createRequire(import.meta.url).resolve(specifier)` (the `#` +import map works through `require.resolve` too), with `node:module` and `node:url` loaded through +`process.getBuiltinModule` so it stays synchronous and the root keeps no static `node:` import. +`test/package/import-meta.test.ts` scans `out/lib` and `out/node` and refuses any `import.meta` +form but `.url`. The branch was rebased onto the 9.1.0 release commit (`fe53ab7`) so the fix can +ship as 9.1.1. + +XLSX was checked and left alone: the five mailwoman call sites read small workbooks, no workbook +over 5 MB exists on `/mnt/mw`, and `read-excel-file` has no streaming API. + +### 3.13 String sources split as text (2026-10-09) + +`TextSpliterator.from(string)` encoded the string, scanned the bytes and decoded each row back: +7.8 us per call on a 40-line string against 1 us for `split` + `trim`. That is the shape of the ~80 +stdout-splitting call sites in mailwoman. `textRows` (`lib/io/windowed-text.ts`) now splits the +string on the delimiter text with the engine's `crlf`, `skipEmpty`, `drop`, `take` semantics; the +streaming window path shares the same splitter. Gated by `canSplitAsText` (no byte `position`, no +quote handling, UTF-8 only). 2.5 us per call; the remainder is the `Sequence` and the generator. +`JSONSpliterator.from(string)` takes the same route with the text comment filter. +`test/formats/text-rows.test.ts` pins parity with the byte route over 9 texts x 8 option sets. + +### 3.14 Row-count benchmark rows: scan-only and worker-parallel (2026-10-09, kimi session) + +Top item of the updated `HANDOFF_SUGGESTIONS.md` review. The count adapters full-parse (every cell produced and +discarded), because uDSV's do; the library's scan-only path (`CSVSpliterator.countAsync`, which counts engine rows +and never decodes columns) was never shown. Two new adapters, both labeled as their own rows so the full-parse +counts stay for apples-to-apples: + +- `spliterator-rows-count.ts` → "spliterator (stream, row count)": `countAsync(path, { header: false, trim: false, + bulkThreshold: 0 })`. +- `spliterator-parallel-count.ts` → "spliterator (stream, row count, N workers)": `AsyncSpliterator.segments` + (the documented boundary primitive) plus one **persistent** worker thread per core + (`parallel-count-worker.ts`), spawned in `load()` so the timed cycles measure the scan, not the spawn. Each + worker opens its own range with `createChunkIterator({ start, end: end - 1 })` and counts engine rows with + `crlf: true, enableQuoteHandling: true` — the same row semantics as `countAsync`'s streaming branch — and posts + one number back, so a segment of any size costs one message (no per-record transfer; `asManyWorkers` was not + used because its per-record results stream would put 25M items through the merged iterator on gb-tuples). + N is `availableParallelism()` (16 here) and the name carries it. + +Results (single fresh sweep, quiet machine, load ~2; `results.json` and the README tables were regenerated from +it, so every number in the README moved slightly): + +| Dataset | full-parse count | row count | row count, 16 workers | uDSV (stream, count) | +| --------------------- | ---------------- | --------- | --------------------- | -------------------- | +| litmus_quoted 1.8 MiB | 199 | **715** | 669 | 249 | +| openpayments 386 MiB | 285 | **1091** | **2015** | 310 | +| gb-tuples 1.42 GiB | 174 | **393** | **721** | 271 | + +The scan-only sequential row beats uDSV's full-parse count everywhere (1.45–3.5×); the parallel row beats it +2.7–6.5×. On the 1.9 MB litmus file the parallel row _loses_ to the sequential one (669 vs 715 — segment probing +is per call) and costs 474 MiB against 53, which is visible in the table and is the expected small-file trade-off. +Parallel scaling on gb-tuples is only 1.8× (393 → 721) — short rows (~57 B) make it per-row-bound, and `/mnt/mw` +read bandwidth caps the rest. + +**Verification.** Sequential `countAsync`, the rows-count adapter, and the parallel adapter agree exactly on all +three datasets, and match uDSV's `expected.json` where it exists (litmus 10000, openpayments 1697026; gb-tuples +25674049, unverified by the harness but identical across all three). **Known limit, demonstrated:** segments are +quote-blind, so a file with the row delimiter inside a quoted field miscounts in parallel — a synthetic 10 MB file +of `"a\nb",c` rows counted 581251 parallel against 300000 sequential. The adapter header and the README narrative +carry the caveat; all three published datasets are verified per the sentence above. The sound fix is library-level +quote-aware segmentation: the engine toggles quote state per quote byte, so the parity of `"` bytes before a cut +is exactly its quote state — countable per segment in parallel (order-free) and prefix-XORed, with a bounded +forward repair probe for cuts that land odd. Not done; that is what would make `asManyWorkers` safe for CSV with +embedded newlines generally. + +Verification and timing scripts lived at the repo root as `scratch-*.mjs` (gitignored) and were deleted after use — +note that `yarn test`'s compile type-checks even root-level `.mjs` files, so scratch scripts cannot be left lying +around. + +--- + +## 4. Clean before/after summary (worktree of `main` vs. working tree, quiet machine) + +| Path | Before | After | +| ------------------------------------------ | -------- | -------- | +| 300 MB quoted CSV streamed, rows discarded | 4.2 s | ~1.9 s | +| 266 MB unquoted CSV streamed | 3.2 s | 1.9 s | +| 300 MB `TextSpliterator` rows streamed | 2.0 s | 1.6 s | +| 386 MB CSV as a string, in memory | 4.2 s | 3.8 s | +| in-memory peak RSS above baseline | 2237 MiB | 1465 MiB | + +--- + +## 5. Measurement caveats + +- Another session on this machine (a mailwoman checkout) compiles intermittently; load average + reached 15–20 at times. Every number above was retaken after waiting for 1-minute load < 4.5 + (`until awk '{exit !($1 < 4.5)}' /proc/loadavg; do sleep 15; done`). The harness sweep ran at a + steady ~4 that applied equally to every parser. Absolute numbers will shift on a quiet machine; + ratios should hold. Always interleave A/B runs here; sequential blocks drifted by 20%. +- A "before" build cannot be a copied `out/` directory: the `#` import map in `package.json` + resolves relative to the nearest `package.json`, so a copy outside the package fails with + `ERR_PACKAGE_IMPORT_NOT_DEFINED`, and a copy inside it would resolve `#core/*` to the _current_ + `out/`. Use `git worktree add HEAD`, `git diff HEAD | git -C apply`, symlink + `node_modules`, compile there. +- `hasScanner()` is false in a fresh process until `whenReady()` resolves. Any standalone timing + script that constructs a sync `Spliterator`/`CSVSpliterator.from` without awaiting it measures + the JS scanner. + +--- + +## 6. How uDSV gets its speed, and what it gives up + +Read from `../uDSV/src/uDSV.mjs`. + +**Schema inference up front.** `inferSchema` samples the first rows, decides delimiter, whether a +quote character appears anywhere, and the column count. The parser loop in `parse` has the column +count baked in: on every column but the last it does `indexOf(colDelim)`, on the last +`indexOf(rowDelim)`. One `indexOf` and one `slice` per cell, no per-character state machine when +the file has no quote at all (`colEncl === ''` branch, the one that hits ~300 MiB/s). The quoted +branch walks for the closing quote and only `replaceAll`s on a seen escape. + +**Generated row builder.** `genToTypedRow` emits JavaScript source as a string, one expression per +column, and compiles it with `new Function`. For string arrays it collapses to returning the row. +For objects it is a literal `{ id: +r[0], name: r[1], ... }` with keys inlined, so V8 sees a fixed +shape with constant keys instead of a loop doing dynamic property assignment. This is why uDSV's +object mode is nearly as fast as its array mode, where spliterator's object mode costs about 2×. + +**What happens when the schema is wrong: nothing.** There is no check. + +- A short row never reaches the last column index, so the parser searches for a comma, runs through + the newline, and glues the next row's leading cells onto this one. Rows merge until columns + happen to realign. +- A long row hits the last index early, then searches for the newline, so every extra column lands + in the last cell with the commas in it. +- A value that breaks an inferred type goes through the compiled conversion; a word in a numeric + column becomes NaN unless the caller configured a replacement. The typed builder never re-infers. +- The quoted branch chooses the delimiter the same way, so row-merging is identical there. + +None of this raises. The harness only checks row and column counts, which these regular datasets +never trip. On a ragged file spliterator's per-row behaviour is correct and uDSV's is undefined. +That is the honest counterweight to the 2× speed gap and belongs next to it in any write-up. + +**What is portable.** The row builder: object mode in `lib/formats/row-emitters.ts` could compile a +per-header builder once with `new Function` after the header is known; applies to the per-row and +windowed paths alike. Risk: CSP / some bundlers forbid `new Function`, so it needs a fallback to the +current loop. The schema-specialized scan is the expensive one (an inference pass, a no-quotes +kernel, and a policy for a row that breaks the shape) and is effectively a different library. + +--- + +## 7. Where the remaining cost is + +- **Streaming path, string arrays:** after 3.7 the per-cell cost is one `slice`, `normalizeCell`, + and an indexed store, plus the window decode. There is no obvious implementation gap left on this + path; what remains is the layering (byte engine → decode → cell kernel → sequence) versus uDSV's + single string loop. +- **In-memory string path:** after 3.7 the GC share is gone and the remaining litmus gap (~1.7×) + is the kernel round-trip (~30% of the parse on a 2 MB source) plus slicing. On the 386 MB string + the earlier decode removal bought only 10% of time while halving memory. First-row latency is + the whole-source decode on the first pull (~28 ms on 40 MB). +- **Always measure with rows retained.** A discard loop hides scavenger cost entirely (3.7). +- **Object mode** now costs the same as array mode (3.8). Entries mode is ~1.7× array mode, which is + the 3-tuple per cell. +- `AsyncSequence` wrapping is ~1.9× a bare async generator per item; on parsed rows that is 3–8%. + Documented in `AGENTS.md` with the reasons not to restructure it. + +--- + +## 8. WebAssembly string builtins probe (throwaway, 2026-10-09) + +Question: can the kernel create the cell strings itself through the +[JS String Builtins](https://github.com/WebAssembly/js-string-builtins) (`wasm:js-string`) and the +UTF-8 extension (`wasm:text-decoder` / `wasm:text-encoder`, behind +`--experimental-wasm-imported-strings-utf8` on Node 26.2 / V8 14.6), and does that beat slicing in +JavaScript? + +Files, all in the scratchpad `spike/` directory (not in the repo; `binaryen` and `wabt` were npm +installed there for `wasm-as`): `strings.wat` (imports `substring`, exports a loop that writes +substrings into an `externref` table or calls an imported `emit` per cell), `decode.wat` (a GC +`array i8` filled from a string via `encodeStringIntoUTF8Array`, then `decodeStringFromUTF8Array` +per cell), `check.mjs` (functional check), `bench.mjs` (timing). Compile with +`WebAssembly.compile(bytes, { builtins: ['js-string', 'text-decoder', 'text-encoder'] })`. + +Findings: + +1. `wasm:js-string` works on this Node **with no flag**. `wasm:text-decoder` / `text-encoder` need + `--experimental-wasm-imported-strings-utf8`; without it instantiation fails with + `Import #0 "wasm:text-decoder": module is not an object or function`. +2. Timings, 2 million cells, best of 7, boundaries precomputed: + +| Variant | ASCII cells | Mixed UTF-8 cells | +| -------------------------------------------------------------------------------- | ----------- | ----------------- | +| JS `slice` from an `Int32Array` of boundaries (today's cell loop) | 75 ms | 79 ms | +| wasm `substring` into an `externref` table, no readback | 82 ms | 79 ms | +| same, then JS reads the table back with `table.get` | 102 ms | 138 ms | +| wasm `substring` with an imported `emit` callback per cell | 133 ms | 124 ms | +| JS: `TextDecoder.decode` the window once, then `slice` (today, including decode) | 83 ms | 109 ms | +| JS: `TextDecoder.decode` per cell on a `subarray` | 251 ms | 310 ms | +| wasm `decodeStringFromUTF8Array` per cell from a GC array, via `emit` | 156 ms | 173 ms | + +3. Conclusions: + - Creating a string costs the same from either side. The wasm substring loop ties JS only when + the results stay in wasm; any way of getting them out (table read, callback) is a boundary + crossing per cell and loses. + - Decoding per cell loses to decoding the window once even with the builtin, so the current + decode-then-slice design is correct. + - **Structural blocker:** the UTF-8 builtins read only from WebAssembly GC arrays. There is no + instruction or JS API that copies linear memory or a `Uint8Array` into a GC `array i8` + (`array.init_data` copies from a _data segment_; `array.new_fixed` is per element). The only + bulk way in is encoding a JS string, which is circular here. + - Stable Rust cannot express `externref` parameters or GC types, so any of this would be + hand-written `.wat` beside the Rust kernel. + - Net: no path from the string builtins to a faster string-array output. Not pursued. + +--- + +## 9. Ideas not yet tried, roughly ranked + +1. ~~Typed / columnar output from the kernel~~ — dropped 2026-10-09: no consumer in mailwoman + (section 3.10). Revisit only if a numeric workload appears. +2. ~~`new Function` row builder for object mode~~ — done in 3.8. +3. ~~Kernel emits per-row cell counts for preallocation~~ — done differently in 3.7 (previous + row's width as a template), no kernel change needed. +4. **Skip `normalizeCell` when `trim` is off and the cell has no quote.** Check whether V8 already + makes this free before bothering. + +--- + +## 10. Reproducing + +```bash +# repo +yarn compile && yarn test --run && yarn lint + +# kernel (requires wasm32-unknown-unknown target; wasm-opt optional) +bash wasm/build.sh + +# harness sweep (see header of benchmarks/udsv/sweep.ts for uDSV setup) +node out/benchmarks/udsv/sweep.js --stream /mnt/mw/ppd/2026-07-22/gb-tuples.csv +node out/benchmarks/udsv/render.js # Markdown tables from results.json + +# quick stream timing +node /count.mjs /gb300.csv +``` + +Working-tree state: the 3.1–3.13 library work was committed (the branch was rebased onto the 9.1.0 release commit +per 3.12). What remains uncommitted is this session's benchmark work: modified `README.md` (regenerated benchmark +tables + row-count narrative), `benchmarks/udsv/results.json`, `benchmarks/udsv/sweep.ts` (two new entries in +`countParsers`); untracked `benchmarks/udsv/adapters/spliterator-rows-count.ts`, +`benchmarks/udsv/adapters/spliterator-parallel-count.ts`, +`benchmarks/udsv/adapters/parallel-count-worker.ts`; plus the staged `HANDOFF.md`/`HANDOFF_SUGGESTIONS.md` +themselves. The `../uDSV` checkout has only `bench/expected.json` entries added and dataset symlinks in +`bench/data/`. diff --git a/HANDOFF_SUGGESTIONS.md b/HANDOFF_SUGGESTIONS.md new file mode 100644 index 0000000..4b0eb39 --- /dev/null +++ b/HANDOFF_SUGGESTIONS.md @@ -0,0 +1,32 @@ +Read it all. The gap isn't uniform, and that matters for where the drastic moves are: + +| Workload | spliterator | uDSV | Gap | +| ----------------------------- | ----------- | ---- | --------------- | +| litmus in-memory | 103 | 358 | **3.5×** | +| openpayments in-memory | 78 | 125 | 1.6× | +| openpayments stream+count | 212 | 302 | 1.4× | +| gb-tuples stream+count | 127 | 266 | 2.1× | +| openpayments stream, retained | 78 | 76 | **already won** | + +Per handoff §7 the residual cost is per-cell JS work (slice, normalize, push) plus layering — not scanning. uDSV is essentially at the V8 string-creation floor, so no byte-pipeline tweak closes 3.5×. Drastic means one of three things: produce cells cheaper, don't produce cells, or don't produce them on one thread. Ranked: + +**1. Compiled row builders via `new Function` — highest ROI, uDSV-proven.** +`lib/formats/row-emitters.ts:102` builds objects with `zipSync` (an iterator + tuple allocation per cell) and dynamic `record[key] =`. That's why object mode is ~2× array mode. uDSV's `genToTypedRow` compiles `{ id: r[0], name: r[1] }` with constant keys so V8 sees a fixed shape. Compile per-header emitters once the header is known, for object _and_ entries mode, plus a fixed-arity array literal for the common fixed-column case. Applies to row path and windowed path alike. Expected: object mode → near array mode; entries similar; ~10–20% on arrays. Needs a no-`Function` fallback for CSP/bundlers. This is the only move with a directly comparable proof point in uDSV itself. + +**2. In-memory: drop the kernel round-trip, run a fused `indexOf` loop on the string — biggest single gap.** +The 3.5× litmus gap lives here. Current path encodes the string into WASM memory, scans bytes, and slices at UTF-16 boundaries. But your own §8 probe measured JS `slice` at ~37 ns/cell — exactly uDSV's per-cell rate — because V8's `String.prototype.indexOf` is already SIMD. A fused, quote-aware, `indexOf`-driven loop over the string (uDSV's architecture, but with correct ragged-row and quote handling instead of schema inference) should land 200–300+ MiB/s in-memory. Keep the kernel path for byte sources. A few days of spike work; the risk is a per-character JS state machine sneaking in and eating the win — stay `indexOf`-driven like uDSV's quoted branch. + +**3. Don't produce cells for count/aggregate work — and go parallel.** +The count adapters build full string arrays and discard them (`benchmarks/udsv/adapters/spliterator-count.ts`), which is why "count" runs at half of `TextSpliterator`'s row-scan speed (1.6s vs 2.4s on 300 MB). A scan-only count is ~187 MiB/s single-threaded; segment-parallel over `asManyWorkers`/`WorkerPool` multiplies that by cores — 3–5× past uDSV's 266 on gb-tuples, using parts that already exist and are tested. Two honesty constraints: a scan-only count is a _different workload_ than uDSV's full-parse count, so it needs its own labeled row in the table, not a silent adapter swap; and as a product feature (a true `countAsync` fast path, projection, in-worker aggregation) it's real value, not benchmark theater. + +**4. Typed/columnar kernel output — the floor-breaker, and the biggest design cost.** +Handoff §9.1. As long as output is string arrays, you're bounded by V8 string creation and can at best tie uDSV. Parsing numeric columns into `Float64Array` inside the Rust kernel straight from bytes, emitting boundaries only for projected columns, never creates those strings at all — 5–10× on numeric-heavy data is plausible, and it's the one thing a byte-level kernel can do that a string-native parser structurally cannot. This is a new API surface (projection, type hints, columnar result), so it's a design doc before it's code. It's also the most defensible direction for the library's identity: byte ranges, SIMD, bounded memory. + +**5. Sound schema specialization — adopt uDSV's trick without the unsoundness.** +uDSV _infers_ "no quotes" from a sample and corrupts ragged files silently. Spliterator can _prove_ it: one SIMD scan for the quote byte over the whole source (~70 ms on 386 MB at 5–6 GB/s, fusable into the streaming read per window). A proven-no-quote file unlocks: zero quote state anywhere, known column count → preallocated `new Array(n)` rows (have the kernel emit per-row cell counts, §9.3), no `normalizeCell` quote branch, compiled builders from move 1. On its own it's worth less than uDSV's numbers suggest (uDSV's _quoted_ branch also does ~266, so quote handling isn't the main cost) — it's the multiplier on moves 1–2, not a standalone fix. And the ragged-row policy stays correct per row, which is the honest counterweight already in your README. + +**6. Micro, verify first:** skip `normalizeCell` when `trim: false` and no quote flag (check whether V8 already makes it free), kernel-emitted cell counts for preallocation. A few percent each; measure before landing. + +Sequencing: **1 first** (days, broad, certain), then **2** (closes the worst table), then **3** (inverts the count tables with existing machinery). **4** is the only move that changes the game rather than the score — worth a design doc once 1–3 have landed and the remaining gap is measured again. + +Want me to start on one of these? I'd suggest spiking #1 or #2 to validate the expected payoff before committing to the full change. diff --git a/README.md b/README.md index 2940e61..fc25f99 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,9 @@ runner, its adapters for the other parsers, and its timing metric (the geometric three-second cycles, reported as MiB/s). Every parser produces arrays of strings, with no typing, trimming, or header handling. The harness gives each parser a string already in memory; the streaming tables read the file with each parser's own streaming API instead, retaining every row in the `(stream)` rows and discarding them in the -`(stream, count)` rows. +`(stream, count)` rows. The `(stream, row count)` rows are spliterator-only and a different workload: they count +rows without decoding columns — the scan-only path a pre-count or projection takes — so they are labeled apart +from the full-parse counts rather than compared against them. Memory is the process's peak resident set over the whole run, less the baseline the runner reports before loading the parser. An in-memory run therefore includes the input string, which every parser is handed alike, and a retained @@ -403,58 +405,67 @@ disagreed with uDSV's by more than the harness tolerates is listed with that err | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | ----------------- | -------------------------------------------- | -------------------------------------------- | -| csv-simple-parser | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 394 | ░░░░░░░░░░░░ 58 | -| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 372 | ░░░░░░░░░░░░ 58 | -| spliterator | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 288 | ░░░░░░░░░░░░░░░░░░░░ 96 | -| but-csv | ░░░░░░░░░░░░░░░░░░░ 183 | ░░░░░░░░░░░░░░░░░░░░░░░ 112 | -| d3-dsv | ░░░░░░░░░░ 93.8 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | -| PapaParse | ░░░░░░░░ 75 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 192 | +| csv-simple-parser | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 387 | ░░░░░░░░░░░░ 57 | +| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 371 | ░░░░░░░░░░░░ 58 | +| spliterator | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 288 | ░░░░░░░░░░░░░░░░░░░░ 96 | +| but-csv | ░░░░░░░░░░░░░░░░░░░ 187 | ░░░░░░░░░░░░░░░░░░░░░░░ 112 | +| d3-dsv | ░░░░░░░░░░ 95.5 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | +| PapaParse | ░░░░░░░░ 72.7 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 192 | **litmus_quoted.csv (1.8 MiB, 10K rows), streamed from file** -| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | -| --------------------------- | -------------------------------------------- | -------------------------------------------- | -| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 247 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 155 | -| uDSV (stream) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 226 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 196 | -| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 197 | ░░░░░░░░░░░░░░░░░░░ 92 | -| spliterator (stream) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 176 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 182 | -| PapaParse (stream, count) | ░░░░░░░░░░░░░░ 84.8 | ░░░░░░░░░░░░░░ 71 | -| PapaParse (stream) | ░░░░░░░░░░░ 66.5 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 190 | +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| ------------------------------------------- | -------------------------------------------- | -------------------------------------------- | +| spliterator (stream, row count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 715 | ░░░░ 53 | +| spliterator (stream, row count, 16 workers) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 669 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 474 | +| uDSV (stream, count) | ░░░░░░░░░░░░░░ 249 | ░░░░░░░░░░░░░ 153 | +| uDSV (stream) | ░░░░░░░░░░░░░ 231 | ░░░░░░░░░░░░░░░░░ 202 | +| spliterator (stream, count) | ░░░░░░░░░░░ 199 | ░░░░░░░ 84 | +| spliterator (stream) | ░░░░░░░░░░ 178 | ░░░░░░░░░░░░░ 158 | +| PapaParse (stream, count) | ░░░░░ 89.2 | ░░░░░░ 72 | +| PapaParse (stream) | ░░░░ 66 | ░░░░░░░░░░░░░░░░░ 203 | **openpayments_covered-recipient-profile_20260603.csv (386 MiB, 1.7M rows), in-memory string** | Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | | ----------------- | --------------------------------------------------- | ---------------------------------------------- | -| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 143 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.82K | -| d3-dsv | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 103 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.86K | -| spliterator | ░░░░░░░░░░░░░░░░░░░░░░░░ 84.6 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.94K | +| uDSV | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 163 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.82K | +| d3-dsv | ░░░░░░░░░░░░░░░░░░░░░░░░ 95.6 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.86K | +| spliterator | ░░░░░░░░░░░░░░░░░░░░░ 87.2 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.93K | | PapaParse | Wrong row count! Expected: 1697026, Actual: 1697027 | | | csv-simple-parser | Wrong row count! Expected: 1697026, Actual: 1697027 | | | but-csv | Wrong row count! Expected: 1697026, Actual: 1697027 | | **openpayments_covered-recipient-profile_20260603.csv (386 MiB, 1.7M rows), streamed from file** -| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | -| --------------------------- | -------------------------------------------- | ---------------------------------------------- | -| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 314 | ░ 137 | -| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 285 | ░ 92 | -| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░ 191 | ░ 82 | -| spliterator (stream) | ░░░░░░░░░░░░░░ 109 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.37K | -| PapaParse (stream) | ░░░░░░░░░░░ 88 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.71K | -| uDSV (stream) | ░░░░░░░░░ 69.9 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.61K | +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| ------------------------------------------- | ---------------------------------------------- | ---------------------------------------------- | +| spliterator (stream, row count, 16 workers) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 2.01K | ░░░░░░ 575 | +| spliterator (stream, row count) | ░░░░░░░░░░░░░░░░░░░░░░ 1.09K | 33 | +| uDSV (stream, count) | ░░░░░░ 310 | ░ 140 | +| spliterator (stream, count) | ░░░░░░ 285 | ░ 90 | +| PapaParse (stream, count) | ░░░░ 193 | ░ 83 | +| spliterator (stream) | ░░ 119 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.42K | +| PapaParse (stream) | ░░ 93.1 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.75K | +| uDSV (stream) | ░ 70.8 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 3.65K | **gb-tuples.csv (1.42 GiB), streamed from file** -| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | -| --------------------------- | -------------------------------------------- | -------------------------------------------- | -| uDSV (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 277 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 130 | -| spliterator (stream, count) | ░░░░░░░░░░░░░░░░░░░░░░░░ 169 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 97 | -| PapaParse (stream, count) | ░░░░░░░░░░░░░░░░░░ 122 | ░░░░░░░░░░░░░░░░░░░░░░░ 74 | +| Name | Throughput (MiB/s) | Peak RSS above baseline (MiB) | +| ------------------------------------------- | -------------------------------------------- | -------------------------------------------- | +| spliterator (stream, row count, 16 workers) | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 721 | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 532 | +| spliterator (stream, row count) | ░░░░░░░░░░░░░░░░░░░░░░ 393 | ░░░ 46 | +| uDSV (stream, count) | ░░░░░░░░░░░░░░░ 271 | ░░░░░░░░░░░░░░░ 131 | +| spliterator (stream, count) | ░░░░░░░░░░ 174 | ░░░░░░░░ 101 | +| PapaParse (stream, count) | ░░░░░░░ 118 | ░░░░░░ 75 | CSV into arrays of strings is parse-bound, and on that work spliterator is mid-pack: uDSV generates a parser per -schema and slices one decoded string, and nothing byte-oriented will catch that. What this harness cannot show is -what spliterator is for — raw byte-range scanning at SIMD speed, a bounded footprint on a file of any size, and the -chainable sequence over the rows. See [Choosing a primitive](#choosing-a-primitive) for where each of those pays. +schema and slices one decoded string, and nothing byte-oriented will catch that. The row-count rows are what the +same engine does when cells are never produced — the byte-range scanner at SIMD speed, sequential, and then split +across worker threads with `AsyncSpliterator.segments`. Segments align on the record delimiter without regard to +quote state, so the parallel count assumes no quoted field contains a newline; both rows are verified against the +sequential parser on every dataset above. See [Choosing a primitive](#choosing-a-primitive) for where each of +those pays. To rerun: `benchmarks/udsv/sweep.ts` drives the harness from a sibling uDSV checkout and writes `benchmarks/udsv/results.json`; `benchmarks/udsv/render.ts` prints the tables above from it. Setup is in the header diff --git a/benchmarks/udsv/adapters/parallel-count-worker.ts b/benchmarks/udsv/adapters/parallel-count-worker.ts new file mode 100644 index 0000000..aa74a68 --- /dev/null +++ b/benchmarks/udsv/adapters/parallel-count-worker.ts @@ -0,0 +1,62 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * Worker half of `spliterator-parallel-count.ts`. Counts the quote-aware, CRLF-normalized logical rows of one byte + * range — the same row semantics as `CSVSpliterator.countAsync`'s streaming branch — and posts a single number back, + * so a segment of any size costs one message. The WASM scanner is warmed once at module load; workers persist across + * the harness's timing cycles. + */ + +import { parentPort } from "node:worker_threads" + +import { AsyncSpliterator, CharacterSequence } from "spliterator" +import { createChunkIterator } from "spliterator/node/fs" + +export interface CountRowsRequest { + source: string + /** + * Range start byte. + */ + start: number + /** + * Range end byte, exclusive — one past the delimiter that closed the previous record. + */ + end: number +} + +export interface CountRowsReply { + rows?: number + error?: string +} + +const scannerReady = CharacterSequence.whenReady() + +parentPort!.on("message", async (request: CountRowsRequest) => { + const reply = (message: CountRowsReply) => parentPort!.postMessage(message) + + try { + await scannerReady + + const chunks = await createChunkIterator(request.source, { + start: request.start, + end: request.end - 1, + }) + + await using engine = new AsyncSpliterator(chunks, { + crlf: true, + enableQuoteHandling: true, + }) + + let rows = 0 + + for await (const _row of engine) { + rows++ + } + + reply({ rows }) + } catch (error) { + reply({ error: error instanceof Error ? error.message : String(error) }) + } +}) diff --git a/benchmarks/udsv/adapters/spliterator-parallel-count.ts b/benchmarks/udsv/adapters/spliterator-parallel-count.ts new file mode 100644 index 0000000..d2be788 --- /dev/null +++ b/benchmarks/udsv/adapters/spliterator-parallel-count.ts @@ -0,0 +1,90 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: count rows with one worker thread per delimiter-aligned segment + * (`AsyncSpliterator.segments`, the library's documented boundary primitive), each worker scanning its own range at + * engine speed and reporting a single number. Workers persist across the harness's cycles, so the timed parse + * measures the scan, not the spawn. + * + * Segments align to the record delimiter without regard to quote state, so a file carrying the row delimiter inside + * a quoted field can be miscounted. Every published dataset is verified against the sequential `countAsync` before + * its numbers land in the tables (see HANDOFF.md). + */ + +import { availableParallelism } from "node:os" +import { Worker } from "node:worker_threads" + +import { AsyncSpliterator } from "spliterator" + +import type { CountRowsReply, CountRowsRequest } from "./parallel-count-worker.ts" + +const concurrency = availableParallelism() + +export const name = `spliterator (stream, row count, ${concurrency} workers)` +export const repo = "https://github.com/sister-software/spliterator" + +const workers: Worker[] = [] + +export async function load() { + if (!workers.length) { + const workerUrl = new URL("./parallel-count-worker.js", import.meta.url) + + for (let i = 0; i < concurrency; i++) { + workers.push(new Worker(workerUrl)) + } + } + + // Requests are serialized per worker; the harness awaits each parse, so a worker never holds more than one. + const queues = new WeakMap>() + + const send = (worker: Worker, request: CountRowsRequest): Promise => { + const run = (queues.get(worker) ?? Promise.resolve()).then( + () => + new Promise((resolve, reject) => { + const onMessage = (reply: CountRowsReply) => { + worker.off("message", onMessage) + + if (reply.error === undefined) { + resolve(reply.rows ?? 0) + } else { + reject(new Error(reply.error)) + } + } + + worker.on("message", onMessage) + worker.postMessage(request) + }) + ) + + queues.set( + worker, + run.catch(() => {}) + ) + + return run + } + + return async (_csvStr: string, path: string) => { + const segments = await AsyncSpliterator.segments(path, { concurrency }) + + const counts = await Promise.all( + segments.map(([start, end], index) => send(workers[index % workers.length]!, { source: path, start, end })) + ) + + let rows = 0 + + for (const count of counts) { + rows += count + } + + return [[rows]] + } +} + +export async function unload() { + await Promise.all(workers.map((worker) => worker.terminate())) + + workers.length = 0 +} diff --git a/benchmarks/udsv/adapters/spliterator-rows-count.ts b/benchmarks/udsv/adapters/spliterator-rows-count.ts new file mode 100644 index 0000000..885c3c7 --- /dev/null +++ b/benchmarks/udsv/adapters/spliterator-rows-count.ts @@ -0,0 +1,26 @@ +/** + * @license MIT + * @author Teffen Ellis, et al. + * @copyright Sister Software + * + * uDSV bench adapter: stream a file and count rows only, never decoding columns. This is spliterator's scan-only + * path (`CSVSpliterator.countAsync`) — a different workload from the full-parse count adapters, which produce every + * cell and discard it, so it is listed as its own row rather than replacing them. + */ + +import { CSVSpliterator } from "spliterator" + +export const name = "spliterator (stream, row count)" +export const repo = "https://github.com/sister-software/spliterator" + +export async function load() { + return async (_csvStr: string, path: string) => { + const rows = await CSVSpliterator.countAsync(path, { + header: false, + trim: false, + bulkThreshold: 0, + }) + + return [[rows]] + } +} diff --git a/benchmarks/udsv/results.json b/benchmarks/udsv/results.json index 9166a52..189122f 100644 --- a/benchmarks/udsv/results.json +++ b/benchmarks/udsv/results.json @@ -4,7 +4,7 @@ "mib": 1.8, "name": "spliterator", "gmean_ms": 6.4, - "mibs": 287.5, + "mibs": 287.6, "rss_mib": 96, "rows": 10000, "error": null @@ -14,7 +14,7 @@ "mib": 1.8, "name": "uDSV", "gmean_ms": 5, - "mibs": 371.9, + "mibs": 370.6, "rss_mib": 58, "rows": 9999, "error": null @@ -23,8 +23,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "PapaParse", - "gmean_ms": 24.6, - "mibs": 75, + "gmean_ms": 25.4, + "mibs": 72.7, "rss_mib": 192, "rows": 10000, "error": null @@ -33,9 +33,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "csv-simple-parser", - "gmean_ms": 4.7, - "mibs": 393.6, - "rss_mib": 58, + "gmean_ms": 4.8, + "mibs": 387.1, + "rss_mib": 57, "rows": 10000, "error": null }, @@ -43,8 +43,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "d3-dsv", - "gmean_ms": 19.7, - "mibs": 93.8, + "gmean_ms": 19.4, + "mibs": 95.5, "rss_mib": 196, "rows": 10000, "error": null @@ -53,8 +53,8 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "but-csv", - "gmean_ms": 10.1, - "mibs": 183.3, + "gmean_ms": 9.9, + "mibs": 187.2, "rss_mib": 112, "rows": 10000, "error": null @@ -63,9 +63,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "spliterator (stream)", - "gmean_ms": 10.5, - "mibs": 175.8, - "rss_mib": 182, + "gmean_ms": 10.4, + "mibs": 178.4, + "rss_mib": 158, "rows": 10000, "error": null }, @@ -73,9 +73,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "uDSV (stream)", - "gmean_ms": 8.2, - "mibs": 226, - "rss_mib": 196, + "gmean_ms": 8, + "mibs": 231.2, + "rss_mib": 202, "rows": 9999, "error": null }, @@ -83,9 +83,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "PapaParse (stream)", - "gmean_ms": 27.8, - "mibs": 66.5, - "rss_mib": 190, + "gmean_ms": 28, + "mibs": 66, + "rss_mib": 203, "rows": 10000, "error": null }, @@ -93,19 +93,39 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "spliterator (stream, count)", - "gmean_ms": 9.4, - "mibs": 197.2, - "rss_mib": 92, + "gmean_ms": 9.3, + "mibs": 198.8, + "rss_mib": 84, "rows": 2, "error": null }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "spliterator (stream, row count)", + "gmean_ms": 2.6, + "mibs": 715.3, + "rss_mib": 53, + "rows": 1, + "error": null + }, + { + "data": "litmus_quoted.csv", + "mib": 1.8, + "name": "spliterator (stream, row count, 16 workers)", + "gmean_ms": 2.8, + "mibs": 669, + "rss_mib": 474, + "rows": 1, + "error": null + }, { "data": "litmus_quoted.csv", "mib": 1.8, "name": "uDSV (stream, count)", - "gmean_ms": 7.5, - "mibs": 247.4, - "rss_mib": 155, + "gmean_ms": 7.4, + "mibs": 249.3, + "rss_mib": 153, "rows": 2, "error": null }, @@ -113,9 +133,9 @@ "data": "litmus_quoted.csv", "mib": 1.8, "name": "PapaParse (stream, count)", - "gmean_ms": 21.8, - "mibs": 84.8, - "rss_mib": 71, + "gmean_ms": 20.7, + "mibs": 89.2, + "rss_mib": 72, "rows": 2, "error": null }, @@ -123,9 +143,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "spliterator", - "gmean_ms": 4561.7, - "mibs": 84.6, - "rss_mib": 2942, + "gmean_ms": 4427.6, + "mibs": 87.2, + "rss_mib": 2933, "rows": 1697026, "error": null }, @@ -133,9 +153,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "uDSV", - "gmean_ms": 2698.3, - "mibs": 143.1, - "rss_mib": 3817, + "gmean_ms": 2376.4, + "mibs": 162.5, + "rss_mib": 3823, "rows": 1697025, "error": null }, @@ -163,9 +183,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "d3-dsv", - "gmean_ms": 3732.1, - "mibs": 103.4, - "rss_mib": 2856, + "gmean_ms": 4036.8, + "mibs": 95.6, + "rss_mib": 2859, "rows": 1697026, "error": null }, @@ -183,9 +203,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "spliterator (stream)", - "gmean_ms": 3551.6, - "mibs": 108.7, - "rss_mib": 3366, + "gmean_ms": 3247.3, + "mibs": 118.9, + "rss_mib": 3417, "rows": 1697026, "error": null }, @@ -193,9 +213,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "uDSV (stream)", - "gmean_ms": 5524, - "mibs": 69.9, - "rss_mib": 3610, + "gmean_ms": 5452.1, + "mibs": 70.8, + "rss_mib": 3652, "rows": 1697025, "error": null }, @@ -203,9 +223,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "PapaParse (stream)", - "gmean_ms": 4389.3, - "mibs": 88, - "rss_mib": 3710, + "gmean_ms": 4147.8, + "mibs": 93.1, + "rss_mib": 3749, "rows": 1697026, "error": null }, @@ -213,19 +233,39 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "spliterator (stream, count)", - "gmean_ms": 1352.7, - "mibs": 285.4, - "rss_mib": 92, + "gmean_ms": 1354.5, + "mibs": 285, + "rss_mib": 90, "rows": 2, "error": null }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "spliterator (stream, row count)", + "gmean_ms": 354, + "mibs": 1090.6, + "rss_mib": 33, + "rows": 1, + "error": null + }, + { + "data": "openpayments_covered-recipient-profile_20260603.csv", + "mib": 386.1, + "name": "spliterator (stream, row count, 16 workers)", + "gmean_ms": 191.7, + "mibs": 2014.5, + "rss_mib": 575, + "rows": 1, + "error": null + }, { "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "uDSV (stream, count)", - "gmean_ms": 1230.6, - "mibs": 313.7, - "rss_mib": 137, + "gmean_ms": 1244.5, + "mibs": 310.2, + "rss_mib": 140, "rows": 2, "error": null }, @@ -233,9 +273,9 @@ "data": "openpayments_covered-recipient-profile_20260603.csv", "mib": 386.1, "name": "PapaParse (stream, count)", - "gmean_ms": 2026.4, - "mibs": 190.5, - "rss_mib": 82, + "gmean_ms": 2000.2, + "mibs": 193, + "rss_mib": 83, "rows": 2, "error": null }, @@ -243,19 +283,39 @@ "data": "gb-tuples.csv", "mib": 1454.2, "name": "spliterator (stream, count)", - "gmean_ms": 8620.6, - "mibs": 168.7, - "rss_mib": 97, + "gmean_ms": 8374.3, + "mibs": 173.6, + "rss_mib": 101, "rows": 2, "error": null }, + { + "data": "gb-tuples.csv", + "mib": 1454.2, + "name": "spliterator (stream, row count)", + "gmean_ms": 3698.7, + "mibs": 393.1, + "rss_mib": 46, + "rows": 1, + "error": null + }, + { + "data": "gb-tuples.csv", + "mib": 1454.2, + "name": "spliterator (stream, row count, 16 workers)", + "gmean_ms": 2017.3, + "mibs": 720.8, + "rss_mib": 532, + "rows": 1, + "error": null + }, { "data": "gb-tuples.csv", "mib": 1454.2, "name": "uDSV (stream, count)", - "gmean_ms": 5258, - "mibs": 276.6, - "rss_mib": 130, + "gmean_ms": 5361.2, + "mibs": 271.2, + "rss_mib": 131, "rows": 2, "error": null }, @@ -263,9 +323,9 @@ "data": "gb-tuples.csv", "mib": 1454.2, "name": "PapaParse (stream, count)", - "gmean_ms": 11943.9, - "mibs": 121.7, - "rss_mib": 74, + "gmean_ms": 12354.1, + "mibs": 117.7, + "rss_mib": 75, "rows": 2, "error": null } diff --git a/benchmarks/udsv/sweep.ts b/benchmarks/udsv/sweep.ts index 8e32bab..d0b85dc 100644 --- a/benchmarks/udsv/sweep.ts +++ b/benchmarks/udsv/sweep.ts @@ -88,7 +88,13 @@ const retainedParsers = [ "./streaming/untyped/retained/PapaParse.cjs", ] -const countParsers = [adapters + "spliterator-count.js", adapters + "udsv-count.js", adapters + "papaparse-count.js"] +const countParsers = [ + adapters + "spliterator-count.js", + adapters + "spliterator-rows-count.js", + adapters + "spliterator-parallel-count.js", + adapters + "udsv-count.js", + adapters + "papaparse-count.js", +] // The runner reads the whole file into a string unless the adapter path says otherwise. The count adapters never // need it, and the largest files cannot provide it, so a generated copy of the runner widens that test.