diff --git a/comms/blog/assets/fig1-discovery-by-setting.png b/comms/blog/assets/fig1-discovery-by-setting.png
index 95f4a5e..21d6b67 100644
Binary files a/comms/blog/assets/fig1-discovery-by-setting.png and b/comms/blog/assets/fig1-discovery-by-setting.png differ
diff --git a/comms/blog/assets/fig1-discovery-by-setting.svg b/comms/blog/assets/fig1-discovery-by-setting.svg
index dcc3f51..11bf5d1 100644
--- a/comms/blog/assets/fig1-discovery-by-setting.svg
+++ b/comms/blog/assets/fig1-discovery-by-setting.svg
@@ -6,9 +6,9 @@
- LLM in loop
+ LLM involved
- plain libFuzzer
+ no LLM at all
@@ -35,7 +35,7 @@
tree-sitter grammars
- hybrid (LLM in loop)
+ LLM-scaffolded survey (no LLM in loop)10
diff --git a/comms/blog/assets/fig4-ablation.png b/comms/blog/assets/fig4-ablation.png
new file mode 100644
index 0000000..dc61a4a
Binary files /dev/null and b/comms/blog/assets/fig4-ablation.png differ
diff --git a/comms/blog/assets/fig4-ablation.svg b/comms/blog/assets/fig4-ablation.svg
new file mode 100644
index 0000000..b7b9484
--- /dev/null
+++ b/comms/blog/assets/fig4-ablation.svg
@@ -0,0 +1,50 @@
+
diff --git a/comms/blog/figures.py b/comms/blog/figures.py
new file mode 100644
index 0000000..9becd95
--- /dev/null
+++ b/comms/blog/figures.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Regenerate blog figures from sweep output.
+
+Usage (from repo root, needs the sweep JSONL present):
+
+ uv run --project scaffold python comms/blog/figures.py
+
+Emits SVG into ``comms/blog/assets/`` and rasterizes to PNG at 2x via
+ImageMagick (which delegates to librsvg here). The committed PNGs are 2x the
+viewBox, i.e. ``-density 192``.
+
+Only fig4 is generated from data. fig1-fig3 describe the earlier 118-grammar
+survey and the ARVO patching sweep, whose raw outputs are not on this machine
+(``scaffold/results/*`` is gitignored); those SVGs remain hand-maintained.
+
+Style constants below are copied from the existing hand-authored figures so a
+generated figure sits beside them without looking foreign.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+ASSETS = Path(__file__).resolve().parent / "assets"
+LLM_DIR = ROOT / "scaffold" / "results" / "treesitter-llm"
+CTL_DIR = ROOT / "scaffold" / "results" / "treesitter-baseline"
+
+GRAMMARS = ["nushell-nu", "gren", "foam", "typst", "sql"]
+REPS = ["rep1", "rep2", "rep3"]
+
+# --- house style ----------------------------------------------------------- #
+FONT = (
+ "-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, "
+ "Helvetica, Arial, sans-serif"
+)
+BG, BORDER = "#fcfcfb", "#e6e5e1"
+INK, MUTED, FAINT = "#0b0b0b", "#52514e", "#8a8983"
+GRID, AXIS = "#ededea", "#c9c8c4"
+BLUE, ORANGE = "#2a78d6", "#eb6834"
+
+
+def _reps_with_bug() -> dict[str, dict[str, int]]:
+ """{grammar: {arm: n_reps_where_a_memory_safety_bug_was_found}}."""
+ sys.path.insert(0, str(ROOT / "scaffold" / "src"))
+ from parser_security_eval.treesitter.baseline_compare import load_sweep
+
+ out: dict[str, dict[str, int]] = {}
+ for g in GRAMMARS:
+ out[g] = {}
+ for arm, base in (("llm", LLM_DIR), ("control", CTL_DIR)):
+ hits = 0
+ for rep in REPS:
+ gc = load_sweep(base / rep).get(g)
+ if gc and gc.mem_hashes():
+ hits += 1
+ out[g][arm] = hits
+ return out
+
+
+def _llm_only_bugs() -> dict[str, int]:
+ """Distinct memory-safety hashes an arm found that the other never did."""
+ sys.path.insert(0, str(ROOT / "scaffold" / "src"))
+ from parser_security_eval.treesitter.baseline_compare import load_sweep
+
+ llm, ctl = load_sweep(LLM_DIR), load_sweep(CTL_DIR)
+ out = {}
+ for g in GRAMMARS:
+ a = llm[g].mem_hashes() if g in llm else set()
+ b = ctl[g].mem_hashes() if g in ctl else set()
+ if a - b:
+ out[g] = len(a - b)
+ return out
+
+
+def fig4() -> str:
+ reps = _reps_with_bug()
+ only = _llm_only_bugs()
+
+ x0, unit = 250, 120 # x = x0 + reps*unit ; 3 reps -> 360px
+ top, pitch, bar_h = 84, 40, 11
+ bottom = top + pitch * len(GRAMMARS) - (pitch - 2 * bar_h - 4)
+
+ p: list[str] = []
+ p.append(
+ f'")
+ return "\n".join(p) + "\n"
+
+
+def main() -> None:
+ if not LLM_DIR.exists() or not CTL_DIR.exists():
+ sys.exit(f"sweep output missing: need {LLM_DIR} and {CTL_DIR}")
+ svg_path = ASSETS / "fig4-ablation.svg"
+ svg_path.write_text(fig4(), encoding="utf-8")
+ print(f"wrote {svg_path}")
+ rasterize(svg_path)
+
+
+def rasterize(svg_path: Path) -> None:
+ """SVG -> 2x PNG, matching the committed figures' resolution."""
+ png = svg_path.with_suffix(".png")
+ subprocess.run(
+ ["magick", "-density", "192", "-background", "none", str(svg_path), str(png)],
+ check=True,
+ )
+ print(f"wrote {png}")
+
+
+if __name__ == "__main__":
+ main()
+
+
+def _load_records(path: Path) -> list[dict]: # pragma: no cover - helper
+ return [json.loads(x) for x in path.read_text().splitlines() if x.strip()]
diff --git a/comms/blog/main.md b/comms/blog/main.md
index 74ad0d2..a886c89 100644
--- a/comms/blog/main.md
+++ b/comms/blog/main.md
@@ -1,6 +1,6 @@
# Unclear that neurosymbolic fuzzing (fuzzers and LLMs uplifting each other) works very well
-**TL;DR.** We fuzzed 118 real-world [tree-sitter](https://tree-sitter.github.io/tree-sitter/) grammars, found memory-safety bugs in 9 of them (10 distinct crashes, all in hand-written `scanner.c`) plus parse hangs and OOMs in a dozen more, disclosed to maintainers, and are landing fixes upstream — LLM-drafted, crash-replay-verified, [first one merged](https://github.com/fwcd/tree-sitter-kotlin/pull/279). That's the contribution. The finding is a negative result on the question we were funded to answer: nothing we ran supports the idea that an LLM in the fuzzing loop finds more bugs. On OSS-Fuzz-hardened parsers our LLM-in-the-loop agent found zero. On tree-sitter, the discovery side never needed an LLM at all — and a plain-[libFuzzer](https://llvm.org/docs/LibFuzzer.html) control found the same class of bugs, and more of them. The LLM earned its keep writing scaffolding and patching crashes, not finding them.
+**TL;DR.** We fuzzed 118 real-world [tree-sitter](https://tree-sitter.github.io/tree-sitter/) grammars, found memory-safety bugs in 9 of them (10 distinct crashes, all in hand-written `scanner.c`) plus parse hangs and OOMs in a dozen more, disclosed to maintainers, and are landing fixes upstream — LLM-drafted, crash-replay-verified, [first one merged](https://github.com/fwcd/tree-sitter-kotlin/pull/279). That's the contribution. On the question we were funded to answer, the result is mostly but no longer entirely negative. On OSS-Fuzz-hardened parsers our LLM-in-the-loop agent found zero bugs. On tree-sitter our 118-grammar survey never had an LLM in the discovery loop at all, and a plain-[libFuzzer](https://llvm.org/docs/LibFuzzer.html) control found the same class of bugs and more of them. We then went back and ran the ablation we had failed to run — a genuine LLM-in-the-loop arm against plain libFuzzer at equal fuzz-seconds, five grammars, three reps each. There the LLM arm did win, narrowly: it found each grammar's known bug in 3 reps of 3 where the bare fuzzer managed 1 of 3, and it found one leak the control never found in three attempts. That is a real effect on a small sample, and it is much weaker than "AI-accelerated fuzzers self-improving at runtime" implies.
Recently, [MaxvH wrote](https://www.lesswrong.com/posts/KKE6bL8LEpb6KuZWA/funding-formal-methods-for-the-cyberpocalypse) that we ought to develop an FM tool of the future:
@@ -36,12 +36,34 @@ To measure the LLM's contribution to discovery, we ran a control: plain libFuzze
The confession: we designed this as treatment versus control for LLM discovery uplift, and only during write-up did we work out — by asking each other what the code actually does — that the "treatment" arm had no LLM in its discovery loop either. So this comparison is not an ablation of anything. It's two conventional fuzzing configurations that differ in engine setup, and the crashes unique to each side (4 and 8) look like ordinary run-to-run fuzzing variance, which is exactly what two no-LLM runs should produce. We weren't scientific here, and we're saying so rather than dressing the tally up as a result.
-But note what the accident leaves intact: across both rounds, hard targets and soft, **at no point did an LLM contribute to finding a bug.** On hardened parsers the genuine LLM-in-the-loop agent found zero. On soft parsers every bug was found by a conventional fuzzer, and a second conventional fuzzer found the same class of bugs and more. The discovery value came from fuzzing at all, and from pointing the fuzzer at overlooked targets.
+But note what the accident leaves intact: up to this point, across both rounds, hard targets and soft, **no LLM had contributed to finding a bug.** On hardened parsers the genuine LLM-in-the-loop agent found zero. On soft parsers every bug was found by a conventional fuzzer, and a second conventional fuzzer found the same class of bugs and more. The discovery value came from fuzzing at all, and from pointing the fuzzer at overlooked targets.
+## Round 3: running the ablation we owed
+
+Having admitted we hadn't run the experiment, we ran it. The treatment arm this time is a real LLM-in-the-loop discovery agent: it writes the libFuzzer harness, fuzzes a 300-second window, reads back coverage and crash feedback, rewrites the harness, and repeats for five windows. The control is plain libFuzzer with a fixed template harness, given the *same* number of windows of the *same* length against the same corpus — so the only difference is who writes the harness. Five grammars where the survey had already found bugs, three repetitions each: 150 windows and 12.5 hours of fuzzing, which took about 15 hours of wall-clock once the model's harness-writing is counted. That gap is itself a cost the equal-fuzz-seconds framing hides — the LLM arm needed roughly 30% more real time to consume the same fuzzing budget.
+
+Two things had to be fixed before the comparison meant anything, and they are worth naming because both would have produced a confident wrong answer. First, libFuzzer aborts the process on the first crashing input unless you run it in fork mode; our treatment arm didn't, so every "300-second" window was ending after two or three executions with coverage never sampled. The control arm *did* run in fork mode. Any number from that pairing would have measured a libFuzzer flag, not an LLM. Second, our crash-identity hash included line numbers, so one leaking function reached from two of its own call sites counted as two bugs.
+
+With those fixed: **6 memory-safety bugs for the LLM arm, 5 for the control, one found only by the LLM arm.** The headline overlap is 5 of 6, and on a five-grammar sample that difference alone would be unremarkable.
+
+
+
+The replicated effect is **reliability, not capability**. On nushell-nu, gren and foam the LLM arm found the bug in all three reps; the bare fuzzer found it in one of three. These are bugs plain libFuzzer clearly *can* find — it found each of them at least once — but at equal budget it finds them inconsistently. Anyone comparing a single run against a single run here would get a coin flip, which is roughly what our 118-grammar tally in the previous section was measuring.
+
+The one result that looks like genuine capability is a leak in [tree-sitter-sql](https://github.com/DerekStride/tree-sitter-sql): `scan_dollar_string_tag` calls `add_char` and never frees the tag buffer. The LLM arm found it in 3 reps of 3. The control found it in 0 of 3, across roughly 68,000 executions. Reaching it needs a structurally valid `$tag$...$tag$` dollar-quoted string, which is the kind of thing a model that knows SQL writes into its seeds and random mutation of a generic corpus does not stumble into. That is the mechanism the funded hypothesis predicts, and it is the only instance of it we have.
+
+Coverage does not tell a clean story. The LLM arm reached more edges on four of five grammars, but the fixed template beat it on nushell-nu in both usable reps (5,111 vs 4,247 mean edges) — and nushell-nu is one of the grammars where the template found the bug least often. Coverage and bug-finding come apart, which is worth remembering before optimising for either.
+
+We are not going to oversell five grammars. The targets were chosen *because* the earlier survey had already found bugs in them, so this measures re-discovery of known bugs, not discovery of new ones; the sql leak may well be one of the two sql bugs the survey already had. One model, one target class, three reps. What we will say is that this is the first time in the project an LLM in the loop measurably beat the fuzzer without it, and that the margin is a reliability multiplier plus one bug — not a step change.
+
[^overlap]: "Same bug" is judged by a normalized stack hash — top frames reduced to function + scanner-file:line, with absolute paths, binary load offsets, and tree-sitter runtime line numbers stripped. Our first attempt hashed raw frames and reported a spurious 0/10 overlap: the two sweeps ran under different home directories against different runtime commits, so every hash differed even for provably identical crashes (byte-identical [ASan](https://clang.llvm.org/docs/AddressSanitizer.html) summaries). The 6/10 figure is after that fix.
## Takeaway
-"AI-accelerated fuzzers that self-improve their harnesses at runtime" is a natural idea, and we'd guess versions of it get funded more than once. Our experience: on hardened targets the LLM-in-the-loop agent added nothing, because the fuzzer alone had already sufficed years ago; on soft targets we found real bugs without ever needing an LLM in the loop. The niche where LLM *discovery* uplift has to live — targets too hard for vanilla fuzzing but tractable with LLM-guided harnesses — is one we looked for on hard targets and never reached, and never needed on soft ones. We can't rule out that a better agent elicits it; on our evidence, the money is better spent buying CPU for the fuzzer and pointing it at the parsers nobody bothered to fuzz. Where the LLM does earn its keep is everything around the fuzzer: writing the scaffolding, triaging the crashes, and drafting the fixes we're now landing upstream — a different and more defensible claim than the one we set out to test.
+"AI-accelerated fuzzers that self-improve their harnesses at runtime" is a natural idea, and we'd guess versions of it get funded more than once. Our experience: on hardened targets the LLM-in-the-loop agent added nothing, because the fuzzer alone had already sufficed years ago; on soft targets we found real bugs without ever needing an LLM in the loop, and then, when we finally ran the controlled version, the LLM arm won by a modest margin.
+
+The niche where LLM *discovery* uplift has to live — targets too hard for vanilla fuzzing but tractable with LLM-guided harnesses — turns out to exist, and to be narrow. We saw exactly one bug in it. Mostly what the LLM bought was consistency: finding in three runs of three what the bare fuzzer found in one of three. That is worth something if you fuzz a target once and move on, and worth much less if you were going to run the fuzzer three times anyway, which is cheaper than three agent runs. On our evidence the money still mostly goes to CPU for the fuzzer and to pointing it at parsers nobody bothered to fuzz — but "mostly" is doing more work in that sentence than it was before we ran the ablation.
+
+The clearest lesson is methodological, and it is the one we'd press on anyone else funding this. We spent a round comparing two configurations that we believed differed by an LLM and did not. When we built the real comparison, two implementation details — a missing libFuzzer flag on one arm, and line numbers in a crash hash — were each independently sufficient to produce a confident wrong answer in *either* direction. The effect being measured here is small enough that the measurement apparatus has to be more trustworthy than the effect, and ours initially wasn't. Where the LLM most clearly earns its keep remains everything around the fuzzer: writing the scaffolding, triaging the crashes, and drafting the fixes we're now landing upstream.
diff --git a/homelab-control.sh b/homelab-control.sh
new file mode 100755
index 0000000..8e8fc54
--- /dev/null
+++ b/homelab-control.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env nix-shell
+#! nix-shell scaffold/treesitter-shell.nix -i bash
+#
+# Control arm: plain libFuzzer, fixed template harness, NO LLM. Budget and window
+# cadence are derived per-rep from the treatment sweep it is pointed at, so the
+# two arms are equal-fuzz-seconds by construction.
+#
+# Run directly -- ./homelab-control.sh -- NOT `bash homelab-control.sh`.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+# No LLM in this arm, so no API key is strictly needed; sourced anyway to keep
+# the two run scripts symmetric.
+set -a
+# shellcheck disable=SC1091
+source .env
+set +a
+
+export SSL_CERT_FILE="${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}"
+
+cd scaffold
+# -j 1 on purpose: the treatment fuzzed one grammar at a time on one core. Running
+# the control concurrently would give each process less CPU than its treatment
+# counterpart got, biasing the comparison toward the LLM arm.
+uv run parser-security-eval treesitter baseline \
+ --hybrid results/treesitter-llm \
+ --out results/treesitter-baseline \
+ -j 1
diff --git a/homelab-treatment.sh b/homelab-treatment.sh
new file mode 100755
index 0000000..1f82932
--- /dev/null
+++ b/homelab-treatment.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env nix-shell
+#! nix-shell scaffold/treesitter-shell.nix -i bash
+#
+# Homelab ablation run: LLM-in-loop fuzzing vs plain libFuzzer (tree-sitter).
+# See docs/homelab-ablation-run.md for prerequisites and the methodology note
+# about matching the fuzz budget before comparing against the 300s control.
+#
+# Run it directly -- ./homelab-run.sh -- NOT `bash homelab-run.sh`, which skips
+# the shebang and so never enters the nix-shell. The shebang's relative path is
+# resolved against this script's directory, so any cwd works.
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+# Monorepo-root dotenv (ANTHROPIC_API_KEY et al). `set -a` exports every var it
+# defines; a plain `source` would leave them as unexported shell locals.
+set -a
+# shellcheck disable=SC1091
+source .env
+set +a
+
+# NixOS ships no default CA path for Python's OpenSSL, so every urllib fetch
+# (grammar metadata, wiki scrape) dies with CERTIFICATE_VERIFY_FAILED -- and
+# survey/sources.py swallows it into an empty result rather than erroring.
+export SSL_CERT_FILE="${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}"
+
+cd scaffold
+uv run parser-security-eval treesitter llm-fuzz \
+ -g nushell-nu,gren,foam,typst,sql \
+ -m anthropic:claude-opus-5 \
+ --window 300 --max-iterations 5 --reps 3 \
+ --out-dir results/treesitter-llm
diff --git a/scaffold/src/parser_security_eval/treesitter/baseline.py b/scaffold/src/parser_security_eval/treesitter/baseline.py
index 920c227..57058c5 100644
--- a/scaffold/src/parser_security_eval/treesitter/baseline.py
+++ b/scaffold/src/parser_security_eval/treesitter/baseline.py
@@ -31,7 +31,7 @@
import threading
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, as_completed
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from pathlib import Path
from parser_security_eval.treesitter import runtime, triage
@@ -39,6 +39,7 @@
GrammarTarget,
Tier,
TSCrash,
+ TSFuzzResult,
TSLoopIteration,
)
from parser_security_eval.treesitter.runtime import (
@@ -63,6 +64,29 @@ class BaselineTarget:
target: GrammarTarget
fuzz_seconds: int
+ # Per-window seconds, mirroring the hybrid's iteration structure. The hybrid
+ # restarts libFuzzer between harness rewrites, and a restart re-seeds from the
+ # corpus — worth real coverage. Running the control as one long process would
+ # therefore confound "the LLM rewrote the harness" with "restarted N times",
+ # so the control replays the same cadence. Empty => a single fuzz_seconds run.
+ windows: list[int] = field(default_factory=list)
+
+ def schedule(self) -> list[int]:
+ """The window lengths to actually run (never empty)."""
+ return [w for w in self.windows if w > 0] or [self.fuzz_seconds]
+
+
+def rep_dirs(results_dir: Path) -> list[Path]:
+ """Return the per-rep subdirs of a hybrid sweep, or the dir itself if flat.
+
+ ``treesitter llm-fuzz --reps N`` writes ``rep1/``..``repN/`` subdirs (and only
+ writes flat when ``reps == 1``). A control matched against the parent dir would
+ glob zero ``*.jsonl`` and silently fuzz nothing, so callers must expand to reps
+ first and match each one separately — summing reps into a single budget would
+ over-grant the control by a factor of N.
+ """
+ reps = sorted(p for p in results_dir.glob("rep*") if p.is_dir())
+ return reps or [results_dir]
def targets_from_results(results_dir: Path) -> list[BaselineTarget]:
@@ -78,7 +102,8 @@ def targets_from_results(results_dir: Path) -> list[BaselineTarget]:
out: list[BaselineTarget] = []
for path in sorted(results_dir.glob("*.jsonl")):
first: dict[str, object] | None = None
- walltime = 0
+ walltime = 0.0
+ windows: list[int] = []
built_any = False
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
@@ -89,7 +114,17 @@ def targets_from_results(results_dir: Path) -> list[BaselineTarget]:
if rec.get("built"):
built_any = True
fuzz = rec.get("fuzz") or {}
- walltime += int(fuzz.get("duration_seconds") or 0)
+ if not fuzz:
+ continue
+ # elapsed_seconds is what the window really cost; duration_seconds is
+ # only the configured cap and overstates any window that exited early.
+ spent = fuzz.get("elapsed_seconds")
+ if spent is None:
+ spent = fuzz.get("duration_seconds") or 0
+ spent = float(spent)
+ if spent > 0:
+ walltime += spent
+ windows.append(int(round(spent)))
if first is None or not built_any or walltime <= 0:
continue
out.append(
@@ -100,12 +135,55 @@ def targets_from_results(results_dir: Path) -> list[BaselineTarget]:
tier=Tier(first["tier"]),
language=str(first.get("language") or ""),
),
- fuzz_seconds=walltime,
+ fuzz_seconds=int(round(walltime)),
+ windows=windows,
)
)
return out
+def _run_windows(
+ runner: LibFuzzerRunner, windows: list[int]
+) -> tuple[TSFuzzResult, list[Path]]:
+ """Fuzz in successive windows against one corpus; merge the stats.
+
+ Mirrors the hybrid's restart cadence (see ``BaselineTarget.windows``). The
+ corpus dir carries across windows, so later windows resume from what earlier
+ ones found — the same thing that happens on the hybrid side between harness
+ rewrites.
+ """
+ per_window: list[TSFuzzResult] = []
+ crashes: list[Path] = []
+ for w in windows:
+ res, found = runner.run(w)
+ per_window.append(res)
+ crashes.extend(found)
+
+ execs = [r.total_executions for r in per_window if r.total_executions is not None]
+ elapsed = [r.elapsed_seconds for r in per_window if r.elapsed_seconds is not None]
+ total_elapsed = sum(elapsed) if elapsed else None
+ total_execs = sum(execs) if execs else None
+ merged = TSFuzzResult(
+ duration_seconds=sum(r.duration_seconds for r in per_window),
+ elapsed_seconds=total_elapsed,
+ total_executions=total_execs,
+ execs_per_sec=(
+ total_execs / total_elapsed
+ if total_execs is not None and total_elapsed
+ else None
+ ),
+ # Coverage is a high-water mark, not a sum: the windows share a corpus, so
+ # adding them would count the same edges once per window.
+ coverage_pcs=max((r.coverage_pcs for r in per_window), default=0),
+ corpus_size=per_window[-1].corpus_size if per_window else None,
+ crashes_found=sum(r.crashes_found for r in per_window),
+ timed_out=any(r.timed_out for r in per_window),
+ oom_killed=any(r.oom_killed for r in per_window),
+ raw_log_tail=per_window[-1].raw_log_tail if per_window else "",
+ )
+ return merged, crashes
+
+
def _classify_without_replay(crash_file: Path) -> str | None:
"""Return a synthetic sanitizer report for hang/OOM artifacts, else None.
@@ -155,6 +233,7 @@ def run_baseline_grammar(
target: GrammarTarget,
*,
fuzz_seconds: int,
+ windows: list[int] | None = None,
cache_dir: Path = DEFAULT_CACHE,
out_dir: Path = DEFAULT_OUT_DIR,
max_len: int = 65536,
@@ -200,16 +279,17 @@ def run_baseline_grammar(
runtime.gather_seeds(grammar_dir, corpus)
# Fork mode keeps discovering distinct crashes past the first one — the fair
- # analogue of the hybrid patching bug #1 to reach bug #2.
- extra = ["-fork=1", "-ignore_crashes=1"] if fork else []
+ # analogue of the hybrid patching bug #1 to reach bug #2. The flags live on
+ # LibFuzzerRunner itself (fork_workers), so the hybrid arm gets the identical
+ # treatment; passing them through extra_flags here would double them up.
runner = LibFuzzerRunner(
binary=Path(build_result.binary_path or ""),
corpus_dir=corpus,
crashes_dir=crashes,
max_len=max_len,
- extra_flags=extra,
+ fork_workers=1 if fork else 0,
)
- fuzz_res, new_crashes = runner.run(fuzz_seconds)
+ fuzz_res, new_crashes = _run_windows(runner, windows or [fuzz_seconds])
binary = Path(build_result.binary_path or "")
distinct, capped = _distinct_crashes(binary, new_crashes)
@@ -265,6 +345,7 @@ def _run_one(
return run_baseline_grammar(
bt.target,
fuzz_seconds=bt.fuzz_seconds,
+ windows=bt.schedule(),
cache_dir=cache_dir,
out_dir=out_dir,
max_len=max_len,
diff --git a/scaffold/src/parser_security_eval/treesitter/baseline_compare.py b/scaffold/src/parser_security_eval/treesitter/baseline_compare.py
index 4dbdb69..218b889 100644
--- a/scaffold/src/parser_security_eval/treesitter/baseline_compare.py
+++ b/scaffold/src/parser_security_eval/treesitter/baseline_compare.py
@@ -21,7 +21,6 @@
from pathlib import Path
from parser_security_eval.treesitter.models import BugClass
-from parser_security_eval.treesitter.triage import stack_hash
# Resource-exhaustion classes are DoS-ish, share a generic stack hash, and are not
# the memory-safety bugs the writeup is about. Kept in the per-grammar tables but
@@ -29,6 +28,64 @@
_RESOURCE_CLASSES = {BugClass.oom.value, BugClass.timeout.value}
+# A crash only counts as a finding *about the target* if its stack actually
+# entered target code: the grammar's own sources, or the tree-sitter runtime
+# driving them. Frames confined to the generated harness plus libc/libFuzzer mean
+# the harness overflowed its own buffer before parsing anything.
+#
+# This is not cosmetic. Only the LLM arm writes its own harness, so only the LLM
+# arm can produce self-inflicted crashes — they would land wholly in the
+# ``hybrid_only`` column and read as discovery uplift. Filtering here (rather than
+# at fuzz time) keeps it retroactive: the frames are already persisted.
+_TARGET_PATH_MARKERS = (
+ "/src/parser.c",
+ "/src/scanner.c",
+ "/src/scanner.cc",
+ "tree-sitter/lib/src/",
+)
+_TARGET_SYMBOL_PREFIXES = ("ts_", "tree_sitter_")
+
+
+# Frames belonging to the sanitizer/libFuzzer crash machinery or bare libc. A
+# stack made *only* of these is an unwind failure (libFuzzer's "deadly signal"
+# path commonly reports just its own handler), not evidence about where the bug
+# is — so it must not be judged as a harness bug. Confirmed empirically: typst's
+# c6f014c21c9d has a handler-only stack and was found by BOTH arms, and the
+# control cannot produce harness bugs because it never writes a harness.
+_HANDLER_MARKERS = (
+ "__sanitizer_",
+ "fuzzer::",
+ "FuzzerLoop.cpp",
+ "PrintStackTrace",
+ "CrashCallback",
+ "/libc.so",
+ "libc.so.6",
+)
+
+
+def is_handler_only(frames: list[str]) -> bool:
+ """True if every frame is sanitizer/libFuzzer/libc scaffolding."""
+ if not frames:
+ return False
+ return all(any(m in f for m in _HANDLER_MARKERS) for f in frames)
+
+
+def reaches_target(frames: list[str]) -> bool:
+ """True if any frame is in grammar source or the tree-sitter runtime.
+
+ Frames are ``":"``. Matching on either the path or the
+ symbol keeps genuine bugs that unwind entirely through the runtime, while
+ excluding stacks that never leave ``harness.c``/libc/libFuzzer.
+ """
+ for frame in frames:
+ if any(marker in frame for marker in _TARGET_PATH_MARKERS):
+ return True
+ symbol = frame.split(" ", 1)[0]
+ if symbol.startswith(_TARGET_SYMBOL_PREFIXES):
+ return True
+ return False
+
+
def _is_memory_safety(bug_class: str, in_scanner: bool) -> bool:
if bug_class in _RESOURCE_CLASSES:
return False
@@ -38,6 +95,20 @@ def _is_memory_safety(bug_class: str, in_scanner: bool) -> bool:
return True
+def function_key(frames: list[str]) -> str:
+ """Stable key from the frames' *symbols only*, dropping file:line.
+
+ ``stack_hash`` includes line numbers, so one leaking function reached from two
+ call sites inside itself hashes as two bugs (sql's ``scan_dollar_string_tag``
+ at scanner.c:67 and :72 are the same defect). Collapsing to symbols fixes the
+ overcount. The trade-off is that two genuinely distinct bugs in one large
+ function now collide; for hand-written tree-sitter scanners — small functions,
+ few of them — that is the rarer error.
+ """
+ symbols = [f.split(" ", 1)[0] for f in frames if f.strip()]
+ return "|".join(symbols)
+
+
@dataclass
class GrammarCrashes:
"""Distinct crashes a sweep found for one grammar, keyed by stack hash."""
@@ -45,6 +116,13 @@ class GrammarCrashes:
grammar: str
# stack_hash -> (bug_class, in_scanner)
crashes: dict[str, tuple[str, bool]] = field(default_factory=dict)
+ # Crashes dropped by :func:`reaches_target` — kept (not discarded) so the
+ # report can state how many were excluded instead of silently shrinking.
+ harness_only: dict[str, tuple[str, bool]] = field(default_factory=dict)
+ # Real crashes whose stack could not be attributed (handler-only unwind).
+ # Counted as findings — dropping them loses genuine bugs — but tracked apart
+ # so a report can say how much of the total rests on unattributed stacks.
+ unattributed: set[str] = field(default_factory=set)
def mem_hashes(self) -> set[str]:
return {
@@ -55,10 +133,34 @@ def all_hashes(self) -> set[str]:
return set(self.crashes)
+def rep_names(results_dir: Path) -> list[str]:
+ """Names of the per-rep subdirs in a sweep dir, or ``[]`` if it is flat.
+
+ ``--reps N`` writes ``rep1/``..``repN/``; ``--reps 1`` writes flat. Both the
+ hybrid and the (budget-matched) baseline mirror the same layout.
+ """
+ return sorted(p.name for p in results_dir.glob("rep*") if p.is_dir())
+
+
+def sweep_files(results_dir: Path) -> list[Path]:
+ """Every ``.jsonl`` in a sweep, flat layout or ``rep*/`` layout.
+
+ Globbing only the top level would silently return nothing for a ``--reps N``
+ sweep, which reads as "this arm found no bugs" rather than as an error.
+ """
+ return sorted(results_dir.glob("*.jsonl")) + sorted(
+ results_dir.glob("rep*/*.jsonl")
+ )
+
+
def load_sweep(results_dir: Path) -> dict[str, GrammarCrashes]:
- """Load a sweep dir into ``{grammar: GrammarCrashes}`` (deduped by stack hash)."""
+ """Load a sweep dir into ``{grammar: GrammarCrashes}`` (deduped by stack hash).
+
+ Reps are *pooled*: a bug counts as found by the arm if any rep found it. Use
+ :func:`compare_by_rep` for the per-rep breakdown.
+ """
out: dict[str, GrammarCrashes] = {}
- for path in sorted(results_dir.glob("*.jsonl")):
+ for path in sweep_files(results_dir):
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
@@ -77,13 +179,27 @@ def load_sweep(results_dir: Path) -> dict[str, GrammarCrashes]:
# makes old and new logs comparable. Frameless crashes (timeout/OOM,
# excluded from the memory-safety headline) keep their stored hash.
frames = crash.get("top_frames") or []
- h = stack_hash(frames) if frames else (crash.get("stack_hash") or "")
+ # Key on symbols, not file:line, so one function reached from two of
+ # its own call sites is one bug (see :func:`function_key`). Falls back
+ # to the line-sensitive hash only when there are no frames at all.
+ h = function_key(frames) if frames else (crash.get("stack_hash") or "")
if not h:
continue
- gc.crashes[h] = (
+ entry = (
str(crash.get("bug_class") or BugClass.unknown.value),
bool(crash.get("in_scanner")),
)
+ # Frameless crashes (timeout/OOM) have no stack to judge and are
+ # already outside the memory-safety headline; leave them alone.
+ if frames and not reaches_target(frames):
+ if is_handler_only(frames):
+ # Unwind failure, not a harness bug: keep it as a finding.
+ gc.crashes[h] = entry
+ gc.unattributed.add(h)
+ else:
+ gc.harness_only[h] = entry
+ else:
+ gc.crashes[h] = entry
return out
@@ -109,6 +225,14 @@ def baseline_only(self) -> set[str]:
@dataclass
class ComparisonReport:
per_grammar: list[GrammarComparison]
+ # Crashes excluded by :func:`reaches_target`, per arm. Reported, not hidden:
+ # a large hybrid number means the model kept writing broken harnesses, which
+ # is itself a result about the treatment.
+ hybrid_harness_only: int = 0
+ baseline_harness_only: int = 0
+ # Counted in the totals above, but resting on stacks we could not attribute.
+ hybrid_unattributed: int = 0
+ baseline_unattributed: int = 0
@property
def hybrid_mem_total(self) -> int:
@@ -143,7 +267,54 @@ def compare(hybrid_dir: Path, baseline_dir: Path) -> ComparisonReport:
if not h and not b:
continue
per_grammar.append(GrammarComparison(grammar=g, hybrid_mem=h, baseline_mem=b))
- return ComparisonReport(per_grammar=per_grammar)
+ return ComparisonReport(
+ per_grammar=per_grammar,
+ hybrid_harness_only=sum(len(g.harness_only) for g in hybrid.values()),
+ baseline_harness_only=sum(len(g.harness_only) for g in baseline.values()),
+ hybrid_unattributed=sum(len(g.unattributed) for g in hybrid.values()),
+ baseline_unattributed=sum(len(g.unattributed) for g in baseline.values()),
+ )
+
+
+def compare_by_rep(
+ hybrid_dir: Path, baseline_dir: Path
+) -> list[tuple[str, ComparisonReport]]:
+ """Compare rep-by-rep, so the overlap gets a spread instead of one number.
+
+ Pairing repN against repN is meaningful because the control's budget is
+ derived per-rep from the matching treatment rep (see
+ ``baseline.targets_from_results``), so the two are equal-fuzz-seconds by
+ construction. Only reps present in *both* sweeps are paired; a rep the
+ baseline has not run yet is skipped rather than scored as "baseline found
+ nothing", which would fabricate uplift.
+ """
+ h_reps, b_reps = rep_names(hybrid_dir), rep_names(baseline_dir)
+ shared_reps = [r for r in h_reps if r in b_reps]
+ if not shared_reps:
+ return [("pooled", compare(hybrid_dir, baseline_dir))]
+ return [(rep, compare(hybrid_dir / rep, baseline_dir / rep)) for rep in shared_reps]
+
+
+def format_rep_reports(reports: list[tuple[str, ComparisonReport]]) -> str:
+ """Per-rep overlap counts plus the spread, for the pooled table's header."""
+ if len(reports) <= 1:
+ return ""
+ lines = [f"{'rep':10} {'hybrid':>6} {'baseln':>6} {'shared':>6} {'hyb-only':>8}"]
+ lines.append("-" * 40)
+ for rep, rep_report in reports:
+ lines.append(
+ f"{rep:10} {rep_report.hybrid_mem_total:>6} "
+ f"{rep_report.baseline_mem_total:>6} {rep_report.shared_total:>6} "
+ f"{rep_report.hybrid_only_total:>8}"
+ )
+ lines.append("-" * 40)
+ only = [r.hybrid_only_total for _, r in reports]
+ lines.append(
+ f"hybrid-only across reps: min={min(only)} max={max(only)} "
+ f"(n={len(only)} reps). A result that holds in one rep and not the others "
+ f"is noise, not uplift."
+ )
+ return "\n".join(lines)
def format_report(report: ComparisonReport) -> str:
@@ -174,4 +345,18 @@ def format_report(report: ComparisonReport) -> str:
f"{report.hybrid_only_total} found only by the hybrid, "
f"{report.baseline_only_total} only by the baseline."
)
+ if report.hybrid_harness_only or report.baseline_harness_only:
+ lines.append("")
+ lines.append(
+ f"Excluded as harness bugs (stack never entered target code): "
+ f"{report.hybrid_harness_only} hybrid, "
+ f"{report.baseline_harness_only} baseline. Only the hybrid writes its "
+ f"own harness, so these would otherwise count as hybrid-only uplift."
+ )
+ if report.hybrid_unattributed or report.baseline_unattributed:
+ lines.append(
+ f"Included but unattributed (handler-only stack, unwind failed): "
+ f"{report.hybrid_unattributed} hybrid, "
+ f"{report.baseline_unattributed} baseline."
+ )
return "\n".join(lines)
diff --git a/scaffold/src/parser_security_eval/treesitter/cli.py b/scaffold/src/parser_security_eval/treesitter/cli.py
index 18fad8f..c66d883 100644
--- a/scaffold/src/parser_security_eval/treesitter/cli.py
+++ b/scaffold/src/parser_security_eval/treesitter/cli.py
@@ -229,6 +229,7 @@ def baseline(
from datetime import datetime, timezone
from parser_security_eval.treesitter.baseline import (
+ rep_dirs,
run_baseline_sweep,
targets_from_results,
)
@@ -237,10 +238,16 @@ def baseline(
if not hybrid.exists():
typer.echo(f"Hybrid results dir not found: {hybrid}", err=True)
raise typer.Exit(1)
- targets = targets_from_results(hybrid)
- if not targets:
+ # `llm-fuzz --reps N` writes rep1/..repN/ subdirs; each rep gets its own
+ # budget-matched control so both arms end up with the same number of
+ # independent samples (i.e. error bars on both sides, not just the treatment).
+ reps = rep_dirs(hybrid)
+ plan = [(d, targets_from_results(d)) for d in reps]
+ plan = [(d, t) for d, t in plan if t]
+ if not plan:
typer.echo(f"No fuzzable grammars found under {hybrid}", err=True)
raise typer.Exit(1)
+ targets = [t for _, ts in plan for t in ts]
# Tee every progress line to stdout AND a run log, each line timestamped so a
# detached/`tail -f`'d run is legible. The per-grammar JSONL under stays
@@ -298,14 +305,28 @@ def _progress(i: int, total: int, label: str, iters: list[TSLoopIteration]) -> N
)
try:
- run_baseline_sweep(
- targets,
- out_dir=out_dir,
- max_len=max_len,
- fork=not no_fork,
- jobs=jobs,
- progress=_progress,
- )
+ for rep_src, rep_targets in plan:
+ # Mirror the hybrid's layout so baseline-compare can pair rep to rep.
+ rep_out = (
+ out_dir
+ if len(plan) == 1 and rep_src == hybrid
+ else (out_dir / rep_src.name)
+ )
+ rep_out.mkdir(parents=True, exist_ok=True)
+ if len(plan) > 1:
+ _log(
+ f"--- {rep_src.name}: {len(rep_targets)} grammars, "
+ f"{sum(t.fuzz_seconds for t in rep_targets) / 3600:.1f} "
+ f"fuzzing-hours -> {rep_out}"
+ )
+ run_baseline_sweep(
+ rep_targets,
+ out_dir=rep_out,
+ max_len=max_len,
+ fork=not no_fork,
+ jobs=jobs,
+ progress=_progress,
+ )
_log(
f"baseline done: {tally['crash']} grammars crashed "
f"({tally['bugs']} total crashes), {tally['clean']} clean, "
@@ -330,14 +351,31 @@ def baseline_compare(
),
) -> None:
"""Diff hybrid vs baseline sweeps on memory-safety stack hashes."""
- from parser_security_eval.treesitter.baseline_compare import compare, format_report
+ from parser_security_eval.treesitter.baseline_compare import (
+ compare,
+ compare_by_rep,
+ format_rep_reports,
+ format_report,
+ sweep_files,
+ )
for label, path in (("hybrid", hybrid), ("baseline", baseline_dir)):
if not path.exists():
typer.echo(f"{label} dir not found: {path}", err=True)
raise typer.Exit(1)
- report = compare(hybrid, baseline_dir)
- typer.echo(format_report(report))
+ if not sweep_files(path):
+ typer.echo(
+ f"{label} dir has no *.jsonl (checked rep*/ too): {path}", err=True
+ )
+ raise typer.Exit(1)
+
+ # Per-rep first (the spread), then the pooled table (the headline).
+ rep_reports = compare_by_rep(hybrid, baseline_dir)
+ rep_table = format_rep_reports(rep_reports)
+ if rep_table:
+ typer.echo(rep_table)
+ typer.echo("")
+ typer.echo(format_report(compare(hybrid, baseline_dir)))
@app.command("survey")
diff --git a/scaffold/src/parser_security_eval/treesitter/models.py b/scaffold/src/parser_security_eval/treesitter/models.py
index 9aa7f53..d10ab12 100644
--- a/scaffold/src/parser_security_eval/treesitter/models.py
+++ b/scaffold/src/parser_security_eval/treesitter/models.py
@@ -66,7 +66,12 @@ class TSBuildResult(BaseModel):
class TSFuzzResult(BaseModel):
"""Parsed result of one libFuzzer run (one fuzz window)."""
- duration_seconds: int
+ duration_seconds: int # the *configured* window, not what was actually spent
+ # Wall-clock the fuzz process really consumed. libFuzzer can exit long before
+ # -max_total_time (it aborts on the first finding unless in fork mode), so
+ # anything budget-matching one arm against another must use this, not
+ # duration_seconds. None on records written before this field existed.
+ elapsed_seconds: float | None = None
execs_per_sec: float | None = None
corpus_size: int | None = None
total_executions: int | None = None
diff --git a/scaffold/src/parser_security_eval/treesitter/registry.py b/scaffold/src/parser_security_eval/treesitter/registry.py
index edde389..dad1136 100644
--- a/scaffold/src/parser_security_eval/treesitter/registry.py
+++ b/scaffold/src/parser_security_eval/treesitter/registry.py
@@ -85,6 +85,43 @@
language="YAML",
symbol="tree_sitter_yaml",
),
+ # ---- Ablation targets ----------------------------------------------------
+ # Grammars where the original survey found memory-safety crashes; the
+ # LLM-in-loop vs plain-libFuzzer arm fuzzes exactly these five (see
+ # docs/homelab-ablation-run.md). All carry external scanners. Where the wiki
+ # lists competing forks, the chosen owner is pinned in the URL.
+ GrammarTarget(
+ name="nushell-nu",
+ repo_url="https://github.com/nushell/tree-sitter-nu",
+ tier=Tier.less_popular,
+ language="Nu",
+ ),
+ GrammarTarget(
+ name="gren",
+ repo_url="https://github.com/MaeBrooks/tree-sitter-gren",
+ tier=Tier.less_popular,
+ language="Gren",
+ ),
+ GrammarTarget(
+ name="foam",
+ repo_url="https://github.com/FoamScience/tree-sitter-foam",
+ tier=Tier.less_popular,
+ language="FoAM",
+ ),
+ GrammarTarget(
+ name="typst",
+ repo_url="https://github.com/uben0/tree-sitter-typst",
+ tier=Tier.less_popular,
+ language="Typst",
+ ),
+ # Upstream commits no src/grammar.json, so the build must run
+ # `tree-sitter generate` before compiling.
+ GrammarTarget(
+ name="sql",
+ repo_url="https://github.com/DerekStride/tree-sitter-sql",
+ tier=Tier.less_popular,
+ language="SQL",
+ ),
]
REGISTRY: dict[str, GrammarTarget] = {g.name: g for g in _GRAMMARS}
diff --git a/scaffold/src/parser_security_eval/treesitter/runtime.py b/scaffold/src/parser_security_eval/treesitter/runtime.py
index bfb50b8..0254b0e 100644
--- a/scaffold/src/parser_security_eval/treesitter/runtime.py
+++ b/scaffold/src/parser_security_eval/treesitter/runtime.py
@@ -12,6 +12,7 @@
import re
import shutil
import subprocess
+import time
from dataclasses import dataclass, field
from pathlib import Path
@@ -343,6 +344,15 @@ def build(spec: BuildSpec, harness_override: str | None = None) -> TSBuildResult
# cov: is the number of covered edges/PCs — the primary "is the harness reaching
# target code" signal. Take the max seen across the run.
_COV_RE = re.compile(r"\bcov:\s*(\d+)")
+# In -fork mode the parent never emits stat:: lines (those come from the child
+# that -print_final_stats was handed). Its progress lines are the only source of
+# totals, e.g.
+# "#706: cov: 1359 ft: 3673 corp: 77 exec/s: 0 oom/timeout/crash: 0/0/57 time: 20s job: 58"
+# The leading #N is cumulative executions across all jobs; exec/s is printed as 0
+# by the parent, so rate has to be derived from the elapsed "time: Ns" field.
+_FORK_EXECS_RE = re.compile(r"^#(\d+):\s+cov:", re.MULTILINE)
+_FORK_TALLY_RE = re.compile(r"oom/timeout/crash:\s*(\d+)/(\d+)/(\d+)")
+_FORK_TIME_RE = re.compile(r"\btime:\s*(\d+)s")
_CRASH_PREFIXES = ("crash-", "oom-", "timeout-", "leak-")
@@ -357,6 +367,13 @@ class LibFuzzerRunner:
rss_limit_mb: int = 2048
per_input_timeout: int = 20
extra_flags: list[str] = field(default_factory=list)
+ # libFuzzer aborts the whole process on the first crashing input, which makes
+ # -max_total_time a no-op for any harness that crashes early: a 300s window
+ # ends after 2-3 execs with cov: never printed. Fork mode runs each batch in a
+ # child, so a crash kills the child, gets written to -artifact_prefix, and the
+ # parent keeps fuzzing for the full budget. Set to 0 to restore the old
+ # stop-on-first-crash behaviour (e.g. when reproducing a single input).
+ fork_workers: int = 1
def run(self, duration_seconds: int) -> tuple[TSFuzzResult, list[Path]]:
"""Fuzz for *duration_seconds*; return (result, new crash files)."""
@@ -373,27 +390,48 @@ def run(self, duration_seconds: int) -> tuple[TSFuzzResult, list[Path]]:
f"-rss_limit_mb={self.rss_limit_mb}",
f"-timeout={self.per_input_timeout}",
"-print_final_stats=1",
- *self.extra_flags,
]
- safety = duration_seconds + 60
+ if self.fork_workers > 0:
+ # ignore_* are what actually keep the parent alive past a finding;
+ # -fork alone still tears down on the first crash.
+ cmd += [
+ f"-fork={self.fork_workers}",
+ "-ignore_crashes=1",
+ "-ignore_ooms=1",
+ "-ignore_timeouts=1",
+ ]
+ cmd += self.extra_flags
+ # Fork mode's parent has to reap children and merge the corpus after
+ # -max_total_time elapses; with thousands of crash artifacts that teardown
+ # alone can outlast a 60s grace period, and blowing the deadline kills the
+ # run and discards its stats. Give fork runs materially more headroom.
+ safety = duration_seconds + (300 if self.fork_workers > 0 else 60)
run_timed_out = False
+ started = time.monotonic()
try:
res = _run(cmd, timeout=safety)
rc, log = res.returncode, res.stdout + "\n" + res.stderr
except subprocess.TimeoutExpired as exc:
run_timed_out = True
rc = -1
- log = (
- (exc.stdout or b"").decode("utf-8", "replace")
- if isinstance(exc.stdout, bytes)
- else (exc.stdout or "")
- )
+
+ def _text(buf: bytes | str | None) -> str:
+ if isinstance(buf, bytes):
+ return buf.decode("utf-8", "replace")
+ return buf or ""
+
+ # libFuzzer writes its progress and final stats to *stderr*; reading
+ # only stdout here loses the entire log on timeout, which zeroes out
+ # coverage/exec stats for the window (the crash artifacts on disk
+ # survive, so the window looks empty rather than failed).
+ log = _text(exc.stdout) + "\n" + _text(exc.stderr)
after = self._crash_files()
new_crashes = [p for p in after if p not in before]
result = TSFuzzResult(
duration_seconds=duration_seconds,
+ elapsed_seconds=round(time.monotonic() - started, 3),
crashes_found=len(new_crashes),
raw_log_tail=log[-4000:],
corpus_size=sum(1 for _ in self.corpus_dir.iterdir()),
@@ -410,6 +448,21 @@ def run(self, duration_seconds: int) -> tuple[TSFuzzResult, list[Path]]:
result.oom_killed = rc == 137 or "out-of-memory" in log
result.timed_out = run_timed_out or "libFuzzer: timeout" in log
+
+ # Fork-mode fallbacks: fill in whatever the stat:: lines could not supply.
+ fork_execs = [int(x) for x in _FORK_EXECS_RE.findall(log)]
+ if fork_execs and result.total_executions is None:
+ result.total_executions = max(fork_execs)
+ if result.execs_per_sec is None and result.total_executions is not None:
+ elapsed = [int(x) for x in _FORK_TIME_RE.findall(log)]
+ secs = max(elapsed) if elapsed else duration_seconds
+ if secs > 0:
+ result.execs_per_sec = result.total_executions / secs
+ tally = _FORK_TALLY_RE.findall(log)
+ if tally:
+ ooms, timeouts, _crashes = (int(x) for x in tally[-1])
+ result.oom_killed = result.oom_killed or ooms > 0
+ result.timed_out = result.timed_out or timeouts > 0
return result, new_crashes
def _crash_files(self) -> set[Path]: