Align names with task identifiers in eval harnesses

.github/workflows/pages.yml

@@ -26,10 +26,8 @@ jobs:
           node-version: '22'
       - name: Install the native Python library
         run: python3 -m pip install -e .
-      - name: Check scoring, imports, and configuration
-        run: |
-          python3 -m unittest tests.test_data tests.test_engines
-          node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs
+      - name: Run public checks (same command as local development)
+        run: python3 -m tests.check
       - name: Check the installed package without Node on PATH
         run: |
           python3 -m pip wheel . --wheel-dir "$RUNNER_TEMP/quickdash-wheels"
@@ -48,16 +46,6 @@ jobs:
             --weights "$GITHUB_WORKSPACE/configs/weights/oellm.yaml" \
             --eval-set "$GITHUB_WORKSPACE/configs/sets/any-available.yaml" \
             --format json > quickdash-installed-sample.json
-      - name: Check the browser with public fixtures
-        run: |
-          google-chrome --headless --no-sandbox --disable-gpu \
-            --no-first-run --no-default-browser-check \
-            --remote-debugging-port=9227 \
-            --user-data-dir="$RUNNER_TEMP/quickdash-chrome" about:blank \
-            > "$RUNNER_TEMP/quickdash-chrome.log" 2>&1 &
-          quickdash_chrome_pid=$!
-          trap 'kill "$quickdash_chrome_pid" 2>/dev/null || true' EXIT
-          node tests/test_public_browser.mjs || { cat "$RUNNER_TEMP/quickdash-chrome.log"; exit 1; }
       - name: Build shared results and the fictional demo
         run: |
           python3 -m app.build --results-dir results --sample-csv examples/sample-evals.csv \

AGENTS.md

@@ -24,13 +24,19 @@ configuration parsers. Neither implementation is the reference for the other.
   exercises the actual browser calculation module; browser tests check that the
   UI supplies and renders those calculations correctly.
 
-Run the shared contract and scoring checks before publishing:
+Run the full public checks before publishing, using the same command as CI:
 
 ```sh
-python -m unittest tests.test_data tests.test_engines
-node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs
+python3 -m tests.check
 ```
 
+This requires the installed Python package, Node.js 22+, and Chrome. It runs the
+shared Python/JavaScript contract tests and public browser tests, starting and
+stopping an isolated Chrome process automatically. Missing prerequisites fail
+rather than silently skipping browser coverage. Use `CHROME_BIN` for a custom
+Chrome executable. Smaller test commands are useful while developing but do not
+replace this pre-PR check.
+
 See [development and publishing](docs/development.md) for browser checks and
 standalone builds. Keep README setup and common commands accurate, and update the
 relevant guide when public interfaces or scoring policies change.

README.md

@@ -94,14 +94,16 @@ Add `--format json` for a complete report. Diagnostics go to stderr. Python and
 
 ## Development
 
-Application code lives in `app/` and `quickdash/`, tests in `tests/`, and contributor documentation in `docs/`. Common checks:
+Application code lives in `app/` and `quickdash/`, tests in `tests/`, and contributor documentation in `docs/`. Before opening a PR, run the same public checks as CI:
 
 ```sh
-python -m unittest tests.test_data tests.test_engines
-node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs
+python3 -m tests.check
 ```
 
-See [development and publishing](docs/development.md) for the source layout, browser tests, and GitHub Pages workflow.
+This requires the Python package installed in your active environment, Node.js 22+,
+and Chrome. It includes an automatically managed browser session; set `CHROME_BIN`
+for a custom Chrome executable. See [development and publishing](docs/development.md)
+for setup, focused checks, and the GitHub Pages workflow.
 
 ## License
 

configs/evals/lsat_ar.yaml → configs/evals/agieval_lsat_ar.yaml

@@ -1,4 +1,4 @@
-name: LSAT AR
+name: agieval_lsat_ar
 category: Reasoning
 match:
   name: agieval_lsat_ar

configs/evals/arc_challenge.yaml

@@ -1,4 +1,4 @@
-name: ARC Challenge
+name: arc_challenge
 category: Knowledge
 match:
   regex: arc_challenge(?:_mt_.+)?

configs/evals/arc_easy.yaml

@@ -1,4 +1,4 @@
-name: ARC Easy
+name: arc_easy
 category: Knowledge
 match:
   name: arc_easy

configs/evals/belebele.yaml

@@ -1,4 +1,4 @@
-name: Belebele
+name: belebele
 category: Reading
 match:
   regex: belebele_.+

configs/evals/cs_algorithms.yaml → configs/evals/bigbench_cs_algorithms.yaml

@@ -1,4 +1,4 @@
-name: CS Algorithms
+name: bigbench_cs_algorithms
 category: Code
 match:
   name: bigbench_cs_algorithms_generate_until

configs/evals/dyck_languages.yaml → configs/evals/bigbench_dyck_languages.yaml

@@ -1,4 +1,4 @@
-name: Dyck languages
+name: bigbench_dyck_languages
 category: Math
 match:
   name: bigbench_dyck_languages_generate_until

configs/evals/language_id.yaml → configs/evals/bigbench_language_identification.yaml

@@ -1,4 +1,4 @@
-name: Language ID
+name: bigbench_language_identification
 category: Language
 match:
   name: bigbench_language_identification_multiple_choice

configs/evals/operators.yaml → configs/evals/bigbench_operators.yaml

@@ -1,4 +1,4 @@
-name: Operators
+name: bigbench_operators
 category: Math
 match:
   name: bigbench_operators_generate_until

configs/evals/qa_wikidata.yaml → configs/evals/bigbench_qa_wikidata.yaml

@@ -1,4 +1,4 @@
-name: QA Wikidata
+name: bigbench_qa_wikidata
 category: Knowledge
 match:
   name: bigbench_qa_wikidata_generate_until

configs/evals/repeat_copy_logic.yaml → configs/evals/bigbench_repeat_copy_logic.yaml

@@ -1,4 +1,4 @@
-name: Repeat Copy Logic
+name: bigbench_repeat_copy_logic
 category: Math
 match:
   name: bigbench_repeat_copy_logic_generate_until

configs/evals/boolq.yaml

@@ -1,4 +1,4 @@
-name: BoolQ
+name: boolq
 category: Reading
 match:
   name: boolq

configs/evals/commonsenseqa.yaml → configs/evals/commonsense_qa.yaml

@@ -1,4 +1,4 @@
-name: CommonsenseQA
+name: commonsense_qa
 category: Commonsense
 match:
   name: commonsense_qa

configs/evals/copa.yaml

@@ -1,4 +1,4 @@
-name: COPA
+name: copa
 category: Commonsense
 match:
   name: copa

configs/evals/coqa.yaml

@@ -1,4 +1,4 @@
-name: CoQA
+name: coqa
 category: Reading
 match:
   name: coqa

configs/evals/flores200.yaml

@@ -1,4 +1,4 @@
-name: FLORES200
+name: flores200
 category: Translation
 match:
   regex: flores200:.+

configs/evals/global_mmlu.yaml

@@ -1,4 +1,4 @@
-name: Global MMLU
+name: global_mmlu
 category: Knowledge
 match:
   regex: global_mmlu_.+

configs/evals/global_piqa_prompted.yaml

@@ -1,4 +1,4 @@
-name: Global PIQA (prompted)
+name: global_piqa_prompted
 category: Commonsense
 match:
   regex: global_piqa_prompted_.+

configs/evals/gpqa_diamond.yaml → configs/evals/gpqadiamond.yaml

@@ -1,4 +1,4 @@
-name: GPQA Diamond
+name: GPQADiamond
 category: Knowledge
 match:
   name: GPQADiamond

configs/evals/gsm8k.yaml

@@ -1,4 +1,4 @@
-name: GSM8K
+name: gsm8k
 category: Math
 match:
   name: gsm8k

configs/evals/hellaswag.yaml

@@ -1,4 +1,4 @@
-name: HellaSwag
+name: hellaswag
 category: Commonsense
 match:
   regex: hellaswag(?:_.+)?

configs/evals/ifeval.yaml

@@ -1,4 +1,4 @@
-name: IFEval
+name: ifeval
 category: Instruction following
 match:
   name: ifeval

configs/evals/include.yaml → configs/evals/include_base_44.yaml

@@ -1,4 +1,4 @@
-name: INCLUDE
+name: include_base_44
 category: Knowledge
 match:
   regex: include_base_44_.+

configs/evals/jeopardy.yaml

@@ -1,4 +1,4 @@
-name: Jeopardy
+name: jeopardy
 category: Knowledge
 match:
   name: jeopardy

configs/evals/lambada.yaml → configs/evals/lambada_openai.yaml

@@ -1,4 +1,4 @@
-name: LAMBADA
+name: lambada_openai
 category: Reading
 match:
   name: lambada_openai

configs/evals/math_500.yaml → configs/evals/math500.yaml

@@ -1,4 +1,4 @@
-name: MATH-500
+name: MATH500
 category: Math
 match:
   name: MATH500

configs/evals/mbpp.yaml

@@ -1,4 +1,4 @@
-name: MBPP
+name: mbpp
 category: Code
 match:
   name: mbpp

configs/evals/mgsm.yaml

@@ -1,4 +1,4 @@
-name: MGSM
+name: mgsm
 category: Math
 match:
   regex: (?:mgsm_native_cot_.+|global_mgsm_.+)

configs/evals/mmlu.yaml

@@ -1,4 +1,4 @@
-name: MMLU
+name: mmlu
 category: Knowledge
 match:
   regex: mmlu(?:_.+)?

configs/evals/multiblimp.yaml

@@ -1,4 +1,4 @@
-name: MultiBlimp
+name: multiblimp
 category: Language
 match:
   regex: multiblimp_.+

configs/evals/openbookqa.yaml

@@ -1,4 +1,4 @@
-name: OpenBookQA
+name: openbookqa
 category: Knowledge
 match:
   name: openbookqa

configs/evals/opensubtitles.yaml → configs/evals/opensubtitles_multi40.yaml

@@ -1,4 +1,4 @@
-name: OpenSubtitles
+name: opensubtitles_multi40
 category: Translation
 match:
   regex: opensubtitles_.+

configs/evals/piqa.yaml

@@ -1,4 +1,4 @@
-name: PIQA
+name: piqa
 category: Commonsense
 match:
   regex: (?:piqa|global_piqa_completions_.+)

configs/evals/polymath.yaml

@@ -1,4 +1,4 @@
-name: PolyMath
+name: polymath
 category: Reasoning
 match: {regex: 'polymath_.+'}
 metric: exact_match

configs/evals/sib_200.yaml → configs/evals/sib200.yaml

@@ -1,4 +1,4 @@
-name: SIB-200
+name: sib200
 category: Reading
 match:
   regex: sib200_.+

configs/evals/social_iqa.yaml

@@ -1,4 +1,4 @@
-name: Social IQa
+name: social_iqa
 category: Commonsense
 match:
   name: social_iqa

configs/evals/squad_v2.yaml → configs/evals/squadv2.yaml

@@ -1,4 +1,4 @@
-name: SQuAD v2
+name: squadv2
 category: Reading
 match:
   name: squadv2

configs/evals/winogrande.yaml

@@ -1,4 +1,4 @@
-name: WinoGrande
+name: winogrande
 category: Commonsense
 match:
   name: winogrande

configs/evals/wsc273.yaml

@@ -1,4 +1,4 @@
-name: WSC273
+name: wsc273
 category: Commonsense
 match:
   name: wsc273

configs/evals/xcopa.yaml

@@ -1,4 +1,4 @@
-name: XCOPA
+name: xcopa
 category: Commonsense
 match:
   regex: xcopa:.+

configs/evals/x_csqa.yaml → configs/evals/xcsqa.yaml

@@ -1,4 +1,4 @@
-name: X-CSQA
+name: xcsqa
 category: Commonsense
 match:
   regex: xcsqa_.+

configs/sets/any-available.yaml

@@ -2,6 +2,6 @@ version: 1
 name: Any available
 mode: available
 exclude:
-  - Global PIQA (prompted)
+  - global_piqa_prompted
 notes:
   - Compare recognized measurements shared by both models; warn about differences.