Split, cached BigMath-vs-GMP benchmark suite

.gitignore

@@ -1,11 +1,10 @@
-.DS_Store
-.vscode/*
-.vs/*
-a.out
-a.out.*/*
-*.class
-*.in
+build/
+*.o
+*.exe
 *.out
-*.ans
-*.csv
-/build
+*.gz
+*.zip
+*.zip.*
+*.gz.*
+.antigravitycli/
+.benchcache/

CMakeLists.txt

@@ -106,10 +106,25 @@ if(BIGMATH_BUILD_TESTS)
     add_executable(divperf_simple tests/divperf_simple.cpp)
     target_link_libraries(divperf_simple PRIVATE bigmath::bigmath)
 
+    add_executable(regression_bench tests/performance/regression_bench.cpp)
+    target_link_libraries(regression_bench PRIVATE bigmath::bigmath)
+
     # BigDecimal performance bench
     add_executable(bigdecimal_perf tests/bigdecimal_perf.cpp)
     target_link_libraries(bigdecimal_perf PRIVATE bigmath::bigmath)
 
+    # Split benchmark suite — dataset gen + BigMath run (GMP run added below).
+    # Primary interface is tests/performance/benchsuite/run_benchmark.sh, which
+    # compiles these standalone; these targets exist for IDE/CI parity.
+    set(BENCHSUITE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests/performance/benchsuite)
+    add_executable(bench_gen_dataset ${BENCHSUITE_DIR}/gen_dataset.cpp)
+    target_link_libraries(bench_gen_dataset PRIVATE bigmath::bigmath)
+    target_include_directories(bench_gen_dataset PRIVATE ${BENCHSUITE_DIR})
+
+    add_executable(bench_run_bigmath ${BENCHSUITE_DIR}/run_bigmath.cpp)
+    target_link_libraries(bench_run_bigmath PRIVATE bigmath::bigmath)
+    target_include_directories(bench_run_bigmath PRIVATE ${BENCHSUITE_DIR})
+
     # GMP-comparison bench (requires libgmp)
     find_library(GMP_LIBRARY gmp)
     find_path(GMP_INCLUDE_DIR gmp.h)
@@ -121,6 +136,10 @@ if(BIGMATH_BUILD_TESTS)
         target_link_libraries(mfa_vs_gmp_check PRIVATE bigmath::bigmath ${GMP_LIBRARY})
         target_include_directories(mfa_vs_gmp_check PRIVATE ${GMP_INCLUDE_DIR})
         target_include_directories(bench_vs_gmp PRIVATE ${GMP_INCLUDE_DIR})
+
+        add_executable(bench_run_gmp ${BENCHSUITE_DIR}/run_gmp.cpp)
+        target_link_libraries(bench_run_gmp PRIVATE bigmath::bigmath ${GMP_LIBRARY})
+        target_include_directories(bench_run_gmp PRIVATE ${BENCHSUITE_DIR} ${GMP_INCLUDE_DIR})
     else()
         message(STATUS "GMP not found; bench_vs_gmp target skipped")
     endif()

tests/performance/benchsuite/HISTORY.md

@@ -0,0 +1,19 @@
+# BigMath vs GMP — historical comparison
+
+Selected large-size results, one block per run, appended by `run_bigmath`.
+Lower ratio is better (BigMath ms / GMP ms). Rows are the entries flagged
+`historical` in the dataset manifest.
+
+## run 1 — machine=Apple_M1_Max_arm64 profile=quick
+
+| op | size | BigMath ms | GMP ms | BM/GMP | check |
+|----|------|-----------:|-------:|-------:|:-----:|
+| mul | 10000x10000 digits | 0.093 | 0.030 | 3.12x | FAIL |
+| div | 40000x10000 digits | 1.186 | 0.232 | 5.10x | FAIL |
+
+## run 2 — machine=Apple_M1_Max_arm64 profile=quick
+
+| op | size | BigMath ms | GMP ms | BM/GMP | check |
+|----|------|-----------:|-------:|-------:|:-----:|
+| mul | 10000x10000 digits | 0.231 | 0.030 | 7.76x | ok |
+| div | 40000x10000 digits | 1.545 | 0.232 | 6.65x | ok |

tests/performance/benchsuite/README.md

@@ -0,0 +1,49 @@
+# Split BigMath-vs-GMP benchmark suite
+
+`bench_vs_gmp.cpp` regenerates random operands and reruns GMP on every
+invocation, which is slow. This suite splits that into three cacheable stages so
+repeated BigMath runs only pay for BigMath.
+
+## Usage
+
+```
+./run_benchmark.sh [profile] [--force-dataset] [--force-gmp]
+```
+
+`profile` = `quick` | `default` | `full` (default `default`). The driver runs
+only the stages that are stale.
+
+## Stages
+
+1. **Dataset generation** (`gen_dataset`) — materializes random operands once
+   from deterministic per-entry seeds, writes `manifest.csv` + one decimal file
+   per operand, and the driver zips it to `dataset_<profile>.zip`. Skipped if the
+   zip or an extracted copy already exists.
+2. **GMP run** (`run_gmp`) — times GMP on the dataset and records a
+   base-independent value hash for the exact-integer ops. Saved to
+   `gmp_<machine>_<profile>.csv` and reused across BigMath edits, since GMP
+   timings don't change. Runs once per machine per profile.
+3. **BigMath run** (`run_bigmath`) — times BigMath on the same dataset, verifies
+   mul/div against the cached GMP hashes, and prints BigMath/GMP ratios. Saved
+   under an incrementing run id (`bigmath_<machine>_<profile>_<id>.csv`); the
+   `historical`-flagged rows are appended to `HISTORY.md`.
+
+## Layout
+
+- Cache (gitignored): `$BENCH_DIR` (default `<repo>/.benchcache`)
+  - `dataset/<profile>/` + `dataset_<profile>.zip` — operands
+  - `results/` — per-machine GMP + per-run BigMath CSVs
+  - `bin/` — compiled stage binaries
+- Committed: `HISTORY.md` — long-lived comparison table
+
+## Notes
+
+- Operands are reproducible on any machine (fixed seeds), so the dataset is
+  machine-independent; only timings/results are machine-keyed.
+- The value hash is FNV-1a over the integer's little-endian 32-bit word stream
+  (low 32 bits of each base-2^32 BigMath limb; `mpz_export` for GMP), so
+  correctness is checked without the slow decimal `ToString` path.
+- Workload tables live in `workload.h`; the on-disk `manifest.csv` is the source
+  of truth at run time.
+- CMake also exposes `bench_gen_dataset`, `bench_run_gmp`, `bench_run_bigmath`
+  for IDE/CI parity; the shell driver is the primary interface.