diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index c33135b..9db4872 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -26,10 +26,8 @@ jobs: node-version: '22' - name: Install the native Python library run: python3 -m pip install -e . - - name: Check scoring, imports, and configuration - run: | - python3 -m unittest tests.test_data tests.test_engines - node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs + - name: Run public checks (same command as local development) + run: python3 -m tests.check - name: Check the installed package without Node on PATH run: | python3 -m pip wheel . --wheel-dir "$RUNNER_TEMP/quickdash-wheels" @@ -48,16 +46,6 @@ jobs: --weights "$GITHUB_WORKSPACE/configs/weights/oellm.yaml" \ --eval-set "$GITHUB_WORKSPACE/configs/sets/any-available.yaml" \ --format json > quickdash-installed-sample.json - - name: Check the browser with public fixtures - run: | - google-chrome --headless --no-sandbox --disable-gpu \ - --no-first-run --no-default-browser-check \ - --remote-debugging-port=9227 \ - --user-data-dir="$RUNNER_TEMP/quickdash-chrome" about:blank \ - > "$RUNNER_TEMP/quickdash-chrome.log" 2>&1 & - quickdash_chrome_pid=$! - trap 'kill "$quickdash_chrome_pid" 2>/dev/null || true' EXIT - node tests/test_public_browser.mjs || { cat "$RUNNER_TEMP/quickdash-chrome.log"; exit 1; } - name: Build shared results and the fictional demo run: | python3 -m app.build --results-dir results --sample-csv examples/sample-evals.csv \ diff --git a/AGENTS.md b/AGENTS.md index e95ede7..01dddc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,13 +24,19 @@ configuration parsers. Neither implementation is the reference for the other. exercises the actual browser calculation module; browser tests check that the UI supplies and renders those calculations correctly. -Run the shared contract and scoring checks before publishing: +Run the full public checks before publishing, using the same command as CI: ```sh -python -m unittest tests.test_data tests.test_engines -node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs +python3 -m tests.check ``` +This requires the installed Python package, Node.js 22+, and Chrome. It runs the +shared Python/JavaScript contract tests and public browser tests, starting and +stopping an isolated Chrome process automatically. Missing prerequisites fail +rather than silently skipping browser coverage. Use `CHROME_BIN` for a custom +Chrome executable. Smaller test commands are useful while developing but do not +replace this pre-PR check. + See [development and publishing](docs/development.md) for browser checks and standalone builds. Keep README setup and common commands accurate, and update the relevant guide when public interfaces or scoring policies change. diff --git a/README.md b/README.md index a7cd32e..5777281 100644 --- a/README.md +++ b/README.md @@ -94,14 +94,16 @@ Add `--format json` for a complete report. Diagnostics go to stderr. Python and ## Development -Application code lives in `app/` and `quickdash/`, tests in `tests/`, and contributor documentation in `docs/`. Common checks: +Application code lives in `app/` and `quickdash/`, tests in `tests/`, and contributor documentation in `docs/`. Before opening a PR, run the same public checks as CI: ```sh -python -m unittest tests.test_data tests.test_engines -node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs +python3 -m tests.check ``` -See [development and publishing](docs/development.md) for the source layout, browser tests, and GitHub Pages workflow. +This requires the Python package installed in your active environment, Node.js 22+, +and Chrome. It includes an automatically managed browser session; set `CHROME_BIN` +for a custom Chrome executable. See [development and publishing](docs/development.md) +for setup, focused checks, and the GitHub Pages workflow. ## License diff --git a/configs/evals/lsat_ar.yaml b/configs/evals/agieval_lsat_ar.yaml similarity index 97% rename from configs/evals/lsat_ar.yaml rename to configs/evals/agieval_lsat_ar.yaml index 8d606e0..477d5e8 100644 --- a/configs/evals/lsat_ar.yaml +++ b/configs/evals/agieval_lsat_ar.yaml @@ -1,4 +1,4 @@ -name: LSAT AR +name: agieval_lsat_ar category: Reasoning match: name: agieval_lsat_ar diff --git a/configs/evals/arc_challenge.yaml b/configs/evals/arc_challenge.yaml index 5204329..bcbc8f2 100644 --- a/configs/evals/arc_challenge.yaml +++ b/configs/evals/arc_challenge.yaml @@ -1,4 +1,4 @@ -name: ARC Challenge +name: arc_challenge category: Knowledge match: regex: arc_challenge(?:_mt_.+)? diff --git a/configs/evals/arc_easy.yaml b/configs/evals/arc_easy.yaml index afd8f44..28ebb51 100644 --- a/configs/evals/arc_easy.yaml +++ b/configs/evals/arc_easy.yaml @@ -1,4 +1,4 @@ -name: ARC Easy +name: arc_easy category: Knowledge match: name: arc_easy diff --git a/configs/evals/belebele.yaml b/configs/evals/belebele.yaml index eef3a9c..a63a698 100644 --- a/configs/evals/belebele.yaml +++ b/configs/evals/belebele.yaml @@ -1,4 +1,4 @@ -name: Belebele +name: belebele category: Reading match: regex: belebele_.+ diff --git a/configs/evals/cs_algorithms.yaml b/configs/evals/bigbench_cs_algorithms.yaml similarity index 95% rename from configs/evals/cs_algorithms.yaml rename to configs/evals/bigbench_cs_algorithms.yaml index 477ada0..073d064 100644 --- a/configs/evals/cs_algorithms.yaml +++ b/configs/evals/bigbench_cs_algorithms.yaml @@ -1,4 +1,4 @@ -name: CS Algorithms +name: bigbench_cs_algorithms category: Code match: name: bigbench_cs_algorithms_generate_until diff --git a/configs/evals/dyck_languages.yaml b/configs/evals/bigbench_dyck_languages.yaml similarity index 95% rename from configs/evals/dyck_languages.yaml rename to configs/evals/bigbench_dyck_languages.yaml index e925f33..0af699f 100644 --- a/configs/evals/dyck_languages.yaml +++ b/configs/evals/bigbench_dyck_languages.yaml @@ -1,4 +1,4 @@ -name: Dyck languages +name: bigbench_dyck_languages category: Math match: name: bigbench_dyck_languages_generate_until diff --git a/configs/evals/language_id.yaml b/configs/evals/bigbench_language_identification.yaml similarity index 95% rename from configs/evals/language_id.yaml rename to configs/evals/bigbench_language_identification.yaml index de559cc..7f95c90 100644 --- a/configs/evals/language_id.yaml +++ b/configs/evals/bigbench_language_identification.yaml @@ -1,4 +1,4 @@ -name: Language ID +name: bigbench_language_identification category: Language match: name: bigbench_language_identification_multiple_choice diff --git a/configs/evals/operators.yaml b/configs/evals/bigbench_operators.yaml similarity index 96% rename from configs/evals/operators.yaml rename to configs/evals/bigbench_operators.yaml index cca1fbe..a173ece 100644 --- a/configs/evals/operators.yaml +++ b/configs/evals/bigbench_operators.yaml @@ -1,4 +1,4 @@ -name: Operators +name: bigbench_operators category: Math match: name: bigbench_operators_generate_until diff --git a/configs/evals/qa_wikidata.yaml b/configs/evals/bigbench_qa_wikidata.yaml similarity index 96% rename from configs/evals/qa_wikidata.yaml rename to configs/evals/bigbench_qa_wikidata.yaml index 63a39e4..e13417b 100644 --- a/configs/evals/qa_wikidata.yaml +++ b/configs/evals/bigbench_qa_wikidata.yaml @@ -1,4 +1,4 @@ -name: QA Wikidata +name: bigbench_qa_wikidata category: Knowledge match: name: bigbench_qa_wikidata_generate_until diff --git a/configs/evals/repeat_copy_logic.yaml b/configs/evals/bigbench_repeat_copy_logic.yaml similarity index 95% rename from configs/evals/repeat_copy_logic.yaml rename to configs/evals/bigbench_repeat_copy_logic.yaml index 8cbe247..1f7c8e2 100644 --- a/configs/evals/repeat_copy_logic.yaml +++ b/configs/evals/bigbench_repeat_copy_logic.yaml @@ -1,4 +1,4 @@ -name: Repeat Copy Logic +name: bigbench_repeat_copy_logic category: Math match: name: bigbench_repeat_copy_logic_generate_until diff --git a/configs/evals/boolq.yaml b/configs/evals/boolq.yaml index 280b51f..b361742 100644 --- a/configs/evals/boolq.yaml +++ b/configs/evals/boolq.yaml @@ -1,4 +1,4 @@ -name: BoolQ +name: boolq category: Reading match: name: boolq diff --git a/configs/evals/commonsenseqa.yaml b/configs/evals/commonsense_qa.yaml similarity index 97% rename from configs/evals/commonsenseqa.yaml rename to configs/evals/commonsense_qa.yaml index 1ad0330..01ba803 100644 --- a/configs/evals/commonsenseqa.yaml +++ b/configs/evals/commonsense_qa.yaml @@ -1,4 +1,4 @@ -name: CommonsenseQA +name: commonsense_qa category: Commonsense match: name: commonsense_qa diff --git a/configs/evals/copa.yaml b/configs/evals/copa.yaml index 01e34f5..3be34a3 100644 --- a/configs/evals/copa.yaml +++ b/configs/evals/copa.yaml @@ -1,4 +1,4 @@ -name: COPA +name: copa category: Commonsense match: name: copa diff --git a/configs/evals/coqa.yaml b/configs/evals/coqa.yaml index b97ef4b..5bb459f 100644 --- a/configs/evals/coqa.yaml +++ b/configs/evals/coqa.yaml @@ -1,4 +1,4 @@ -name: CoQA +name: coqa category: Reading match: name: coqa diff --git a/configs/evals/flores200.yaml b/configs/evals/flores200.yaml index 0ea1d56..a72eb46 100644 --- a/configs/evals/flores200.yaml +++ b/configs/evals/flores200.yaml @@ -1,4 +1,4 @@ -name: FLORES200 +name: flores200 category: Translation match: regex: flores200:.+ diff --git a/configs/evals/global_mmlu.yaml b/configs/evals/global_mmlu.yaml index 29a2ba4..bb9c9fc 100644 --- a/configs/evals/global_mmlu.yaml +++ b/configs/evals/global_mmlu.yaml @@ -1,4 +1,4 @@ -name: Global MMLU +name: global_mmlu category: Knowledge match: regex: global_mmlu_.+ diff --git a/configs/evals/global_piqa_prompted.yaml b/configs/evals/global_piqa_prompted.yaml index a2a5652..04bfc63 100644 --- a/configs/evals/global_piqa_prompted.yaml +++ b/configs/evals/global_piqa_prompted.yaml @@ -1,4 +1,4 @@ -name: Global PIQA (prompted) +name: global_piqa_prompted category: Commonsense match: regex: global_piqa_prompted_.+ diff --git a/configs/evals/gpqa_diamond.yaml b/configs/evals/gpqadiamond.yaml similarity index 97% rename from configs/evals/gpqa_diamond.yaml rename to configs/evals/gpqadiamond.yaml index 5db1482..3637aba 100644 --- a/configs/evals/gpqa_diamond.yaml +++ b/configs/evals/gpqadiamond.yaml @@ -1,4 +1,4 @@ -name: GPQA Diamond +name: GPQADiamond category: Knowledge match: name: GPQADiamond diff --git a/configs/evals/gsm8k.yaml b/configs/evals/gsm8k.yaml index b0cd554..1b19708 100644 --- a/configs/evals/gsm8k.yaml +++ b/configs/evals/gsm8k.yaml @@ -1,4 +1,4 @@ -name: GSM8K +name: gsm8k category: Math match: name: gsm8k diff --git a/configs/evals/hellaswag.yaml b/configs/evals/hellaswag.yaml index acd22f3..57f32aa 100644 --- a/configs/evals/hellaswag.yaml +++ b/configs/evals/hellaswag.yaml @@ -1,4 +1,4 @@ -name: HellaSwag +name: hellaswag category: Commonsense match: regex: hellaswag(?:_.+)? diff --git a/configs/evals/ifeval.yaml b/configs/evals/ifeval.yaml index 74d77c9..9fab7d2 100644 --- a/configs/evals/ifeval.yaml +++ b/configs/evals/ifeval.yaml @@ -1,4 +1,4 @@ -name: IFEval +name: ifeval category: Instruction following match: name: ifeval diff --git a/configs/evals/include.yaml b/configs/evals/include_base_44.yaml similarity index 99% rename from configs/evals/include.yaml rename to configs/evals/include_base_44.yaml index 6cb8359..1317db5 100644 --- a/configs/evals/include.yaml +++ b/configs/evals/include_base_44.yaml @@ -1,4 +1,4 @@ -name: INCLUDE +name: include_base_44 category: Knowledge match: regex: include_base_44_.+ diff --git a/configs/evals/jeopardy.yaml b/configs/evals/jeopardy.yaml index 108c91c..e024496 100644 --- a/configs/evals/jeopardy.yaml +++ b/configs/evals/jeopardy.yaml @@ -1,4 +1,4 @@ -name: Jeopardy +name: jeopardy category: Knowledge match: name: jeopardy diff --git a/configs/evals/lambada.yaml b/configs/evals/lambada_openai.yaml similarity index 97% rename from configs/evals/lambada.yaml rename to configs/evals/lambada_openai.yaml index 4027348..a960311 100644 --- a/configs/evals/lambada.yaml +++ b/configs/evals/lambada_openai.yaml @@ -1,4 +1,4 @@ -name: LAMBADA +name: lambada_openai category: Reading match: name: lambada_openai diff --git a/configs/evals/math_500.yaml b/configs/evals/math500.yaml similarity index 97% rename from configs/evals/math_500.yaml rename to configs/evals/math500.yaml index d8bc355..5bb477a 100644 --- a/configs/evals/math_500.yaml +++ b/configs/evals/math500.yaml @@ -1,4 +1,4 @@ -name: MATH-500 +name: MATH500 category: Math match: name: MATH500 diff --git a/configs/evals/mbpp.yaml b/configs/evals/mbpp.yaml index c60d1ee..42f2301 100644 --- a/configs/evals/mbpp.yaml +++ b/configs/evals/mbpp.yaml @@ -1,4 +1,4 @@ -name: MBPP +name: mbpp category: Code match: name: mbpp diff --git a/configs/evals/mgsm.yaml b/configs/evals/mgsm.yaml index 206e1fa..857e2d0 100644 --- a/configs/evals/mgsm.yaml +++ b/configs/evals/mgsm.yaml @@ -1,4 +1,4 @@ -name: MGSM +name: mgsm category: Math match: regex: (?:mgsm_native_cot_.+|global_mgsm_.+) diff --git a/configs/evals/mmlu.yaml b/configs/evals/mmlu.yaml index 38b974d..0be53a4 100644 --- a/configs/evals/mmlu.yaml +++ b/configs/evals/mmlu.yaml @@ -1,4 +1,4 @@ -name: MMLU +name: mmlu category: Knowledge match: regex: mmlu(?:_.+)? diff --git a/configs/evals/multiblimp.yaml b/configs/evals/multiblimp.yaml index c1f6c43..3f4b08c 100644 --- a/configs/evals/multiblimp.yaml +++ b/configs/evals/multiblimp.yaml @@ -1,4 +1,4 @@ -name: MultiBlimp +name: multiblimp category: Language match: regex: multiblimp_.+ diff --git a/configs/evals/openbookqa.yaml b/configs/evals/openbookqa.yaml index cf64317..a06c4df 100644 --- a/configs/evals/openbookqa.yaml +++ b/configs/evals/openbookqa.yaml @@ -1,4 +1,4 @@ -name: OpenBookQA +name: openbookqa category: Knowledge match: name: openbookqa diff --git a/configs/evals/opensubtitles.yaml b/configs/evals/opensubtitles_multi40.yaml similarity index 99% rename from configs/evals/opensubtitles.yaml rename to configs/evals/opensubtitles_multi40.yaml index 29888b7..307facb 100644 --- a/configs/evals/opensubtitles.yaml +++ b/configs/evals/opensubtitles_multi40.yaml @@ -1,4 +1,4 @@ -name: OpenSubtitles +name: opensubtitles_multi40 category: Translation match: regex: opensubtitles_.+ diff --git a/configs/evals/piqa.yaml b/configs/evals/piqa.yaml index 48b1f74..dc15c62 100644 --- a/configs/evals/piqa.yaml +++ b/configs/evals/piqa.yaml @@ -1,4 +1,4 @@ -name: PIQA +name: piqa category: Commonsense match: regex: (?:piqa|global_piqa_completions_.+) diff --git a/configs/evals/polymath.yaml b/configs/evals/polymath.yaml index aa9add2..c1df645 100644 --- a/configs/evals/polymath.yaml +++ b/configs/evals/polymath.yaml @@ -1,4 +1,4 @@ -name: PolyMath +name: polymath category: Reasoning match: {regex: 'polymath_.+'} metric: exact_match diff --git a/configs/evals/sib_200.yaml b/configs/evals/sib200.yaml similarity index 99% rename from configs/evals/sib_200.yaml rename to configs/evals/sib200.yaml index 7f99834..b1f6ac1 100644 --- a/configs/evals/sib_200.yaml +++ b/configs/evals/sib200.yaml @@ -1,4 +1,4 @@ -name: SIB-200 +name: sib200 category: Reading match: regex: sib200_.+ diff --git a/configs/evals/social_iqa.yaml b/configs/evals/social_iqa.yaml index f214391..91e606c 100644 --- a/configs/evals/social_iqa.yaml +++ b/configs/evals/social_iqa.yaml @@ -1,4 +1,4 @@ -name: Social IQa +name: social_iqa category: Commonsense match: name: social_iqa diff --git a/configs/evals/squad_v2.yaml b/configs/evals/squadv2.yaml similarity index 97% rename from configs/evals/squad_v2.yaml rename to configs/evals/squadv2.yaml index 57bbe2c..5123936 100644 --- a/configs/evals/squad_v2.yaml +++ b/configs/evals/squadv2.yaml @@ -1,4 +1,4 @@ -name: SQuAD v2 +name: squadv2 category: Reading match: name: squadv2 diff --git a/configs/evals/winogrande.yaml b/configs/evals/winogrande.yaml index 26ed694..81edb9e 100644 --- a/configs/evals/winogrande.yaml +++ b/configs/evals/winogrande.yaml @@ -1,4 +1,4 @@ -name: WinoGrande +name: winogrande category: Commonsense match: name: winogrande diff --git a/configs/evals/wsc273.yaml b/configs/evals/wsc273.yaml index 907ad53..ab34fc6 100644 --- a/configs/evals/wsc273.yaml +++ b/configs/evals/wsc273.yaml @@ -1,4 +1,4 @@ -name: WSC273 +name: wsc273 category: Commonsense match: name: wsc273 diff --git a/configs/evals/xcopa.yaml b/configs/evals/xcopa.yaml index 895430a..46df80e 100644 --- a/configs/evals/xcopa.yaml +++ b/configs/evals/xcopa.yaml @@ -1,4 +1,4 @@ -name: XCOPA +name: xcopa category: Commonsense match: regex: xcopa:.+ diff --git a/configs/evals/x_csqa.yaml b/configs/evals/xcsqa.yaml similarity index 98% rename from configs/evals/x_csqa.yaml rename to configs/evals/xcsqa.yaml index 4dce847..18c3b8b 100644 --- a/configs/evals/x_csqa.yaml +++ b/configs/evals/xcsqa.yaml @@ -1,4 +1,4 @@ -name: X-CSQA +name: xcsqa category: Commonsense match: regex: xcsqa_.+ diff --git a/configs/sets/any-available.yaml b/configs/sets/any-available.yaml index c5d83bd..34e877a 100644 --- a/configs/sets/any-available.yaml +++ b/configs/sets/any-available.yaml @@ -2,6 +2,6 @@ version: 1 name: Any available mode: available exclude: - - Global PIQA (prompted) + - global_piqa_prompted notes: - Compare recognized measurements shared by both models; warn about differences. diff --git a/configs/sets/flagship-1.yaml b/configs/sets/flagship-1.yaml index c78aa06..f2d3c05 100644 --- a/configs/sets/flagship-1.yaml +++ b/configs/sets/flagship-1.yaml @@ -6,50 +6,50 @@ notes: - Require the catalogue tasks for each listed eval, inheriting its scoring defaults. - Exclude Georgian results because of a vocabulary error. Exclude English X-CSQA to avoid overlap with CommonsenseQA. evals: - - name: CS Algorithms + - name: bigbench_cs_algorithms - name: HumanEval - name: LiveCodeBench - - name: MBPP - - name: Dyck languages - - name: Operators - - name: Repeat Copy Logic - - name: GSM8K - - name: MGSM - - name: MATH-500 + - name: mbpp + - name: bigbench_dyck_languages + - name: bigbench_operators + - name: bigbench_repeat_copy_logic + - name: gsm8k + - name: mgsm + - name: MATH500 - name: AIME24 - name: AIME25 - name: AMC23 - name: JEEBench - - name: LSAT AR - - name: PolyMath - - name: ARC Challenge - - name: ARC Easy - - name: GPQA Diamond - - name: INCLUDE - - name: Jeopardy - - name: MMLU - - name: Global MMLU - - name: OpenBookQA - - name: QA Wikidata - - name: CommonsenseQA - - name: COPA - - name: HellaSwag - - name: PIQA - - name: Social IQa - - name: WSC273 - - name: WinoGrande - - name: X-CSQA + - name: agieval_lsat_ar + - name: polymath + - name: arc_challenge + - name: arc_easy + - name: GPQADiamond + - name: include_base_44 + - name: jeopardy + - name: mmlu + - name: global_mmlu + - name: openbookqa + - name: bigbench_qa_wikidata + - name: commonsense_qa + - name: copa + - name: hellaswag + - name: piqa + - name: social_iqa + - name: wsc273 + - name: winogrande + - name: xcsqa exclude_languages: - eng_Latn - - name: XCOPA - - name: Belebele - - name: BoolQ - - name: CoQA - - name: LAMBADA - - name: SIB-200 - - name: SQuAD v2 - - name: FLORES200 - - name: OpenSubtitles - - name: Language ID - - name: MultiBlimp - - name: IFEval + - name: xcopa + - name: belebele + - name: boolq + - name: coqa + - name: lambada_openai + - name: sib200 + - name: squadv2 + - name: flores200 + - name: opensubtitles_multi40 + - name: bigbench_language_identification + - name: multiblimp + - name: ifeval diff --git a/docs/configuration.md b/docs/configuration.md index 701e9cd..acc62e5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,7 +31,7 @@ Both profiles default to the standard calculation and store an English share of **Any available** uses recognized selected measurements shared by A and B. Measurements present on only one side generate comparison warnings and are excluded from both scores. Catalogue entries absent from both models do not generate warnings. The supplied freeform set explicitly excludes prompted Global PIQA pending validation; present data for it generates a **Not used** warning. -**flagship-1** requires the catalogue's known tasks for each listed eval, excluding Georgian throughout the set because of a vocabulary error and English specifically for X-CSQA. Original and translated tasks stay under their existing eval group (for example ARC Challenge). A missing required result generates a warning even if other languages for that eval are present. The score is labelled **INCOMPLETE** and uses the shared subset with redistributed weights. Excluded tasks are not requirements; present excluded data produces **Not used** warnings and remains inspectable. +**flagship-1** requires the catalogue's known tasks for each listed eval, excluding Georgian throughout the set because of a vocabulary error and English specifically for X-CSQA. Original and translated tasks stay under their existing eval group (for example `arc_challenge`). A missing required result generates a warning even if other languages for that eval are present. The score is labelled **INCOMPLETE** and uses the shared subset with redistributed weights. Excluded tasks are not requirements; present excluded data produces **Not used** warnings and remains inspectable. A named set can be concise: @@ -41,12 +41,12 @@ name: Example required set mode: fixed exclude_languages: [kat_Geor] evals: - - name: ARC Challenge + - name: arc_challenge shots: 10 - - name: SIB-200 + - name: sib200 metric: acc metric_filter: none - - name: X-CSQA + - name: xcsqa exclude_languages: [eng_Latn] ``` @@ -66,7 +66,7 @@ Available-mode sets can exclude entire evals or languages without creating requi version: 1 name: Any available mode: available -exclude: [Global PIQA (prompted)] +exclude: [global_piqa_prompted] ``` `exclude` is allowed only in available mode and contains unique, exact catalogue eval names. All names must exist even if no corresponding model results are loaded. Fixed sets exclude unlisted evals automatically. Exclusions warn when eligible task data is present, including rows with only the wrong metric. Alternate metric rows or summary children of a selected eval do not warn merely because another field or summary was selected. diff --git a/docs/development.md b/docs/development.md index 923ef6b..08aecbf 100644 --- a/docs/development.md +++ b/docs/development.md @@ -29,18 +29,36 @@ Check the views affected by your change, including their warnings and failed-inp ## Run tests -These tests use small fixtures and run from a fresh checkout without private evaluation data: +Before opening or updating a PR, run the same public checks as CI from your activated Python environment: ```sh -python -m unittest tests.test_data tests.test_engines +python3 -m tests.check +``` + +Install the package with `python3 -m pip install -e .` first. The command requires +Node.js 22+ and Chrome, runs the Python/JavaScript suites, and launches an isolated +headless Chrome session for the browser tests. It stops Chrome and removes the +temporary profile afterward. Missing Chrome is an error, not a skipped test. +On macOS it discovers the standard Google Chrome application; on Linux it checks +Chrome/Chromium executables on `PATH`. Set `CHROME_BIN` to override the executable. +No private evaluation exports are needed. The command builds local test dashboards +but does not publish a site or push commits. + +For faster iteration, run individual suites: + +```sh +python3 -m unittest tests.test_data tests.test_engines node --test tests/test_data.cjs tests/test_yaml.cjs tests/test_suites.cjs tests/test_warning_policy.cjs tests/test_components.cjs tests/test_view_links.cjs ``` +These narrower commands do not replace the full pre-PR check: catalogue renames, +for example, can pass scoring tests while breaking browser selectors. + The shared suite in `tests/test_engines.py` sends the same input cases to native Python and the real browser engine through `tests/engine_adapter.cjs`. Both must satisfy independently specified expectations, then their complete semantic reports are compared. Diagnostic codes, contexts and actions are checked; presentation text is not. Numerical comparison uses absolute tolerance `1e-9` and relative tolerance `1e-12`. Fixtures vary configuration sizes, contents and ordering. The sample matrix discovers all shipped sets and profiles and exercises every aggregation in both strict and relaxed matching. Shared cases cover override inheritance, language exclusions (including both translation endpoints), invalid references, missing requirements, component completeness, and warning conditions. Browser checks additionally verify effective settings, unchanged catalogue exports, set switching, and failed-import rollback. Python tests also exercise warning emission and CLI stdout/stderr without Node on PATH. They cover input validation, normalization, warning/exclusion behavior, failed-build preservation, Python/JavaScript parity, hierarchy sorting, and deterministic randomized scoring comparisons against an independent calculation. -The public browser suite also checks empty startup, shared models, independent profile/set selection, missing requirements, temporary uploads, rollback, and that file imports make no network requests. It requires Node.js 22+ and Chrome. Start an isolated browser session, then run the suite in another terminal: +The public browser suite also checks empty startup, shared models, independent profile/set selection, missing requirements, temporary uploads, rollback, and that file imports make no network requests. To run just that browser suite manually, start an isolated browser session, then run the suite in another terminal: ```sh "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \ @@ -51,7 +69,7 @@ The public browser suite also checks empty startup, shared models, independent p node tests/test_public_browser.mjs ``` -Adjust the Chrome executable path for your platform. Stop that isolated Chrome process when finished. GitHub Actions runs these public tests automatically. +Adjust the Chrome executable path for your platform. Stop that isolated Chrome process when finished. GitHub Actions runs `python3 -m tests.check` automatically, including this browser suite.
Additional regression tests using the private full export diff --git a/tests/check.py b/tests/check.py new file mode 100644 index 0000000..a5056f8 --- /dev/null +++ b/tests/check.py @@ -0,0 +1,81 @@ +"""Run the public pre-PR checks used by CI, including an isolated browser.""" +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parent.parent +NODE_TESTS = [ + 'test_data.cjs', 'test_yaml.cjs', 'test_suites.cjs', 'test_warning_policy.cjs', + 'test_components.cjs', 'test_view_links.cjs', +] + + +def find_chrome(): + candidates = [os.environ.get('CHROME_BIN'), + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'] + candidates += [shutil.which(name) for name in + ('google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser')] + for candidate in candidates: + if candidate and os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + raise RuntimeError('Chrome is required for the public browser checks. Install Chrome or set CHROME_BIN to its executable.') + + +def run_browser(chrome): + with tempfile.TemporaryDirectory(prefix='quickdash-check-') as temporary: + directory = Path(temporary) + profile = directory / 'profile' + with (directory / 'chrome.log').open('w+') as log: + browser = subprocess.Popen([ + chrome, '--headless', '--no-sandbox', '--disable-gpu', '--no-first-run', + '--no-default-browser-check', '--remote-debugging-port=0', + '--user-data-dir=' + str(profile), 'about:blank', + ], stdout=log, stderr=subprocess.STDOUT) + try: + active_port = profile / 'DevToolsActivePort' + deadline = time.monotonic() + 30 + while not active_port.exists(): + if browser.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError('Chrome did not start its debugging endpoint within 30 seconds.') + time.sleep(.1) + port = int(active_port.read_text().splitlines()[0]) + env = dict(os.environ, QUICKDASH_CHROME_PORT=str(port)) + subprocess.run(['node', 'tests/test_public_browser.mjs'], cwd=ROOT, env=env, check=True) + except Exception: + log.flush() + log.seek(0) + print(log.read(), file=sys.stderr) + raise + finally: + browser.terminate() + try: + browser.wait(timeout=10) + except subprocess.TimeoutExpired: + browser.kill() + browser.wait() + + +def main(): + chrome = find_chrome() # Fail before expensive tests if a prerequisite is missing. + version = subprocess.check_output(['node', '--version'], text=True).strip() + if int(version.lstrip('v').split('.')[0]) < 22: + raise RuntimeError('Node.js 22+ is required for the public browser checks.') + # Browser tests invoke python3; use this command's Python environment for builds. + os.environ['PATH'] = str(Path(sys.executable).parent) + os.pathsep + os.environ.get('PATH', '') + subprocess.run([sys.executable, '-m', 'unittest', 'tests.test_check', + 'tests.test_data', 'tests.test_engines'], cwd=ROOT, check=True) + subprocess.run(['node', '--test'] + ['tests/' + name for name in NODE_TESTS], cwd=ROOT, check=True) + run_browser(chrome) + print('All public checks passed, including the browser.') + + +if __name__ == '__main__': + try: + main() + except (RuntimeError, OSError, subprocess.CalledProcessError) as error: + print('Public checks failed: ' + str(error), file=sys.stderr) + sys.exit(1) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 5746135..3f659e9 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -32,19 +32,19 @@ def test_selected_measurements_unique(self): keys=[tuple(r[k] for k in ['task','metric','filter','n_shot','harness','backend']) for r in rr] self.assertEqual(len(keys),len(set(keys))) def test_summaries_replace_children(self): - rr=[r for r in DATA['rows'] if r['selected'] and r['eval'] in ['MMLU','Global MMLU','INCLUDE']] + rr=[r for r in DATA['rows'] if r['selected'] and r['eval'] in ['mmlu','global_mmlu','include_base_44']] self.assertTrue(all(r['is_group']=='yes' for r in rr)) - self.assertEqual(sum(r['eval']=='MMLU' for r in rr),1) - self.assertEqual(sum(r['eval']=='Global MMLU' for r in rr),16) - self.assertEqual(sum(r['eval']=='INCLUDE' for r in rr),21) + self.assertEqual(sum(r['eval']=='mmlu' for r in rr),1) + self.assertEqual(sum(r['eval']=='global_mmlu' for r in rr),16) + self.assertEqual(sum(r['eval']=='include_base_44' for r in rr),21) def test_scale_and_protocol(self): rr=[r for r in DATA['rows'] if r['selected']] squad=next(r for r in rr if r['task']=='squadv2') self.assertEqual(squad['score_100'],float(squad['value'])) - self.assertTrue(all(r['n_shot']=='0' for r in rr if r['eval']=='HellaSwag')) - self.assertFalse(any('prompted' in r['task'] for r in rr if r['eval']=='PIQA')) + self.assertTrue(all(r['n_shot']=='0' for r in rr if r['eval']=='hellaswag')) + self.assertFalse(any('prompted' in r['task'] for r in rr if r['eval']=='piqa')) def test_missing_eval_is_excluded_and_reweighted(self): - result=summarize([r for r in DATA['rows'] if r['eval']!='IFEval'],CONFIG) + result=summarize([r for r in DATA['rows'] if r['eval']!='ifeval'],CONFIG) self.assertIsNotNone(result[0]['score']) self.assertTrue(next(c for c in result[0]['categories'] if c['name']=='Instruction following')['excluded']) def test_duplicate_rejected(self): @@ -133,13 +133,13 @@ def test_unconfigured_rows_are_excluded(self): self.assertIsNone(result['score_100']);self.assertIn('No eval config',result['decision']) def test_chance_baselines_and_raw_score_preservation(self): evals={e['name']:e for e in CONFIG['evals']} - expected={'SIB-200':1/7,'Language ID':1/11,'Social IQa':1/3,'HellaSwag':.25,'PIQA':.5,'CommonsenseQA':.2,'AIME24':0,'AIME25':0,'JEEBench':.1055,'ARC Easy':.25} + expected={'sib200':1/7,'bigbench_language_identification':1/11,'social_iqa':1/3,'hellaswag':.25,'piqa':.5,'commonsense_qa':.2,'AIME24':0,'AIME25':0,'JEEBench':.1055,'arc_easy':.25} for name,chance in expected.items(): e=evals[name];self.assertEqual(e['normalize']['min'],chance) self.assertAlmostEqual(normalize_score(chance*e['score']['scale'],e)[1],0) self.assertTrue(e['normalize']['sources']) - self.assertEqual(evals['ARC Challenge']['normalize']['min'],.25) - self.assertIn('approximation',evals['ARC Challenge']['normalize']['note']) + self.assertEqual(evals['arc_challenge']['normalize']['min'],.25) + self.assertIn('approximation',evals['arc_challenge']['normalize']['note']) self.assertNotEqual(evals['JEEBench']['normalize'].get('basis'),'unresolved') self.assertEqual(evals['AMC23']['normalize']['min'],0) c=deepcopy(CONFIG) @@ -149,7 +149,7 @@ def test_chance_baselines_and_raw_score_preservation(self): from statistics import mean def raw_eval(name): rr=[r for r in scoped if r['eval']==name] - if name!='PolyMath':return mean(r['raw_score_100'] for r in rr) + if name!='polymath':return mean(r['raw_score_100'] for r in rr) levels={'low':1,'medium':2,'high':4,'top':8} languages={r['task'].split('_')[1] for r in rr} return mean(sum(r['raw_score_100']*levels[r['task'].split('_')[-1]] for r in rr if r['task'].split('_')[1]==lang)/15 for lang in languages) diff --git a/tests/test_app.cjs b/tests/test_app.cjs index 15bc669..a586565 100644 --- a/tests/test_app.cjs +++ b/tests/test_app.cjs @@ -5,7 +5,7 @@ const {scopeRows,inSuite}=require('../app/suite_config.js'); const selected=scopeRows(selectRows(data.rows,scheme),data.suite).rows; assert.ok(selected.length); assert.ok(Math.abs(totals(selected,scheme,scheme.weights).score-data.models[0].score)<1e-10); -const withoutIF=totals(selected.filter(r=>r.eval!=='IFEval'),scheme,scheme.weights); +const withoutIF=totals(selected.filter(r=>r.eval!=='ifeval'),scheme,scheme.weights); assert.ok(Number.isFinite(withoutIF.score));assert.equal(withoutIF.categories.find(c=>c.name==='Instruction following').excluded,true); assert.equal(totals(selected,scheme,{...scheme.weights,Code:.2}).score,null); assert.throws(()=>selectRows([selected[0],selected[0]],scheme),/Duplicate/); @@ -36,13 +36,13 @@ const he=catalogue.find(f=>f.name==='HumanEval').tasks[0]; assert.equal(he.rows.find(r=>r.metric==='python_pass@1').selected,true); assert.equal(he.rows.find(r=>r.metric==='sh_pass@1').selected,false); assert.match(he.rows.find(r=>r.metric==='sh_pass@1').decision,/python_pass@1/); -const hs=catalogue.find(f=>f.name==='HellaSwag').tasks.find(t=>t.name==='hellaswag'); +const hs=catalogue.find(f=>f.name==='hellaswag').tasks.find(t=>t.name==='hellaswag'); assert.equal(hs.rows.length,4); assert.equal(hs.rows.filter(r=>r.selected).length,1); assert.equal(hs.rows.find(r=>r.selected).n_shot,'0'); -const mmlu=catalogue.find(f=>f.name==='MMLU'); +const mmlu=catalogue.find(f=>f.name==='mmlu'); assert.ok(mmlu.tasks.every(t=>!t.name.startsWith('global_'))); -assert.ok(catalogue.find(f=>f.name==='Global MMLU').tasks.every(t=>t.name.startsWith('global_mmlu_'))); +assert.ok(catalogue.find(f=>f.name==='global_mmlu').tasks.every(t=>t.name.startsWith('global_mmlu_'))); assert.ok(mmlu.tasks.some(t=>t.name==='mmlu_abstract_algebra'&&t.rows.every(r=>!r.selected))); const imported=[...data.rows,{...data.rows.find(r=>r.task==='HumanEval'),checkpoint:'second real model'}]; const union=buildCatalogue(auditRows(imported,scheme),scheme); @@ -119,8 +119,8 @@ console.log('Hierarchy checks passed: category/eval/language and language/catego const {diagnosticsFor}=require('./diagnostic_fixture.cjs'); const baselineWarnings=diagnosticsFor(new Map([['real',audited]]),scheme,'standard',{},data.suite); -assert.deepEqual(baselineWarnings.filter(w=>w.type==='Inconsistent scoring settings').map(w=>w.name).sort(),['ARC Challenge','MGSM','PIQA']); -assert.deepEqual(baselineWarnings.filter(w=>w.type==='Config caveat').map(w=>w.name).sort(),['MultiBlimp']); +assert.deepEqual(baselineWarnings.filter(w=>w.type==='Inconsistent scoring settings').map(w=>w.name).sort(),['arc_challenge','mgsm','piqa']); +assert.deepEqual(baselineWarnings.filter(w=>w.type==='Config caveat').map(w=>w.name).sort(),['multiblimp']); const unknown=auditRows([{...data.rows[0],task:'new_task_without_config'}],scheme)[0]; assert.equal(unknown.selected,false);assert.equal(unknown.eval,'');assert.equal(unknown.score_100,null); const warningRows=audited.filter(r=>r.eval!=='HumanEval').concat(unknown); @@ -152,7 +152,7 @@ assert.deepEqual(arcWarnings[0].tasks,['arc_challenge_mt_cs']); assert.match(arcWarnings[0].detail,/expected acc_norm/); const wrongFilter=data.rows.map(r=>r.task==='arc_challenge_mt_cs'&&r.metric==='acc_norm'?{...r,filter:'wrong'}:r); assert.ok(diagnosticsFor(new Map([['real',auditRows(wrongFilter,scheme)]]),scheme).some(w=>w.type==='Missing scoring setting')); -// MMLU child tasks are intentionally excluded, so missing summary metrics there are not warnings. +// mmlu child tasks are intentionally excluded, so missing summary metrics there are not warnings. assert.ok(!diagnosticsFor(new Map([['real',audited]]),scheme).some(w=>w.name==='mmlu_abstract_algebra')); console.log('Column sort and per-task missing metric/setting checks passed.'); // Grouped multilingual scores must expose differences in the selected protocol. @@ -163,19 +163,19 @@ assert.deepEqual(diagnosticsFor(new Map([['real',consistent]]),protocolConfig),[ for(const field of ['n_shot','filter','metric','harness','backend']){ const changed=consistent.map(r=>r.task==='arc_challenge_mt_cs'&&r.selected?{...r,[field]:field==='n_shot'?'5':'different'}:r); const issues=diagnosticsFor(new Map([['real',changed]]),protocolConfig).filter(w=>w.type==='Inconsistent scoring settings'); - assert.equal(issues.length,1,field);assert.equal(issues[0].name,'ARC Challenge');assert.match(issues[0].detail,/arc_challenge_mt_cs/);assert.match(issues[0].detail,/ces_Latn/); + assert.equal(issues.length,1,field);assert.equal(issues[0].name,'arc_challenge');assert.match(issues[0].detail,/arc_challenge_mt_cs/);assert.match(issues[0].detail,/ces_Latn/); } // Alternate, excluded measurements do not change the protocol used in aggregates. const excludedChange=consistent.map(r=>!r.selected?{...r,n_shot:'99'}:r); assert.deepEqual(diagnosticsFor(new Map([['real',excludedChange]]),protocolConfig),[]); -const repeated=consistent.concat(consistent.filter(r=>r.eval==='ARC Challenge'&&r.selected).map(r=>({...r,n_shot:'5'}))); +const repeated=consistent.concat(consistent.filter(r=>r.eval==='arc_challenge'&&r.selected).map(r=>({...r,n_shot:'5'}))); assert.deepEqual(diagnosticsFor(new Map([['real',repeated]]),protocolConfig),[],'same settings set for every task is consistent'); -const noteConfig=structuredClone(protocolConfig);noteConfig.evals.find(e=>e.name==='FLORES200').warning='Review language-pair calibration.'; +const noteConfig=structuredClone(protocolConfig);noteConfig.evals.find(e=>e.name==='flores200').warning='Review language-pair calibration.'; const noteWarnings=diagnosticsFor(new Map([['first',consistent],['second',consistent]]),noteConfig).filter(w=>w.type==='Config caveat'); assert.equal(noteWarnings.length,1,'config notes are not duplicated for each model');assert.match(noteWarnings[0].detail,/language-pair/); assert.throws(()=>validateConfig({...noteConfig,evals:noteConfig.evals.map(e=>({...e,warning:17}))}),/warning/i); console.log('Protocol consistency and editable normalization warning checks passed.'); -for(const name of ['LSAT AR','X-CSQA','Belebele','MultiBlimp'])assert.equal(scheme.evals.find(e=>e.name===name).metric,'acc_norm'); -assert.equal(scheme.evals.find(e=>e.name==='SIB-200').metric,'acc'); -assert.equal(scheme.evals.find(e=>e.name==='SIB-200').warning,undefined); +for(const name of ['agieval_lsat_ar','xcsqa','belebele','multiblimp'])assert.equal(scheme.evals.find(e=>e.name===name).metric,'acc_norm'); +assert.equal(scheme.evals.find(e=>e.name==='sib200').metric,'acc'); +assert.equal(scheme.evals.find(e=>e.name==='sib200').warning,undefined); diff --git a/tests/test_browser.mjs b/tests/test_browser.mjs index 86a94cd..a899b2d 100644 --- a/tests/test_browser.mjs +++ b/tests/test_browser.mjs @@ -58,7 +58,7 @@ assert.equal(await evaluate("document.querySelector('#cards strong').textContent assert.equal(await evaluate("document.querySelectorAll('svg').length"),0); await click('[data-score-category="Reading"]'); assert.equal(await evaluate("document.querySelector('#scoreCategory').value"),'Reading'); -assert.match(await evaluate("document.querySelector('#view').textContent"),/SQuAD v2/); +assert.match(await evaluate("document.querySelector('#view').textContent"),/squadv2/); await evaluate("document.querySelector('#weightEditor').open=true"); await change('[data-weight="Code"]','.2'); assert.equal(await evaluate("document.querySelector('#cards strong').textContent"),'—'); @@ -116,7 +116,7 @@ await click('#clear'); await click('[data-view=comparisons]'); assert.equal(await evaluate("document.querySelector('#compareGroup').value"),'eval'); assert.equal(await evaluate("document.querySelectorAll('.comparison-row').length"),usedEvals); -assert.ok(await evaluate("!!document.querySelector('[data-chart-eval=\"Global MMLU\"]')&&!!document.querySelector('[data-chart-eval=MMLU]')")); +assert.ok(await evaluate("!!document.querySelector('[data-chart-eval=\"global_mmlu\"]')&&!!document.querySelector('[data-chart-eval=mmlu]')")); // Clickable headers sort every displayed value and expose their active direction. for(const [field,index] of [['label',0],['category',1],['a',2],['b',3],['rawDelta',4],['weightedDelta',5],['languageCount',6]]){ for(let clickCount=0;clickCount<2;clickCount++){ @@ -131,21 +131,21 @@ for(const [field,index] of [['label',0],['category',1],['a',2],['b',3],['rawDelt assert.equal(await evaluate("document.querySelector('[data-chart-eval=HumanEval]').cells[1].textContent"),'Code'); // Sections use page scrolling; expanding retains the row position without a nested vertical scroller. await change('#compareSort','name'); -await evaluate(`{const button=document.querySelector('[data-expand-eval="HellaSwag"]'),box=button.closest('.table-scroll');button.scrollIntoView({block:'center'});window.scrollBefore={top:box.scrollTop,left:box.scrollLeft,page:scrollY,button:button.getBoundingClientRect().top};}`); +await evaluate(`{const button=document.querySelector('[data-expand-eval="hellaswag"]'),box=button.closest('.table-scroll');button.scrollIntoView({block:'center'});window.scrollBefore={top:box.scrollTop,left:box.scrollLeft,page:scrollY,button:button.getBoundingClientRect().top};}`); assert.ok(await evaluate('scrollBefore.page>100')); -await click('[data-expand-eval="HellaSwag"]'); -assert.ok(await evaluate(`(()=>{const button=document.querySelector('[data-expand-eval="HellaSwag"]'),box=button.closest('.table-scroll');return Math.abs(button.getBoundingClientRect().top-scrollBefore.button)<2&&Math.abs(scrollY-scrollBefore.page)<2&&box.scrollTop===0})()`),'expansion moved the clicked row'); +await click('[data-expand-eval="hellaswag"]'); +assert.ok(await evaluate(`(()=>{const button=document.querySelector('[data-expand-eval="hellaswag"]'),box=button.closest('.table-scroll');return Math.abs(button.getBoundingClientRect().top-scrollBefore.button)<2&&Math.abs(scrollY-scrollBefore.page)<2&&box.scrollTop===0})()`),'expansion moved the clicked row'); assert.ok(await evaluate("document.querySelectorAll('.comparison-detail').length>1")); assert.match(await evaluate("document.querySelector('.comparison-detail').textContent"),/Latn|Cyrl/); -await click('[data-expand-eval="HellaSwag"]'); +await click('[data-expand-eval="hellaswag"]'); assert.equal(await evaluate("document.querySelectorAll('.comparison-row').length"),usedEvals); -assert.ok(await evaluate(`Math.abs(document.querySelector('[data-expand-eval="HellaSwag"]').getBoundingClientRect().top-scrollBefore.button)<2`),'collapse moved the clicked row'); +assert.ok(await evaluate(`Math.abs(document.querySelector('[data-expand-eval="hellaswag"]').getBoundingClientRect().top-scrollBefore.button)<2`),'collapse moved the clicked row'); assert.equal(await evaluate(`document.querySelector('[data-chart-eval="HumanEval"] .language-count').textContent`),'1'); -assert.ok(await evaluate(`Number(document.querySelector('[data-chart-eval="FLORES200"] .language-count').textContent)>30`)); -assert.equal(await evaluate(`document.querySelector('[data-chart-eval="Language ID"] .language-count').textContent`),'1 pooled'); +assert.ok(await evaluate(`Number(document.querySelector('[data-chart-eval="flores200"] .language-count').textContent)>30`)); +assert.equal(await evaluate(`document.querySelector('[data-chart-eval="bigbench_language_identification"] .language-count').textContent`),'1 pooled'); await change('#language','fin_Latn'); -assert.equal(await evaluate(`document.querySelector('[data-chart-eval="Belebele"] .language-count').textContent`),'1'); -assert.equal(await evaluate(`document.querySelector('[data-chart-eval="FLORES200"] .language-count').textContent`),'2'); +assert.equal(await evaluate(`document.querySelector('[data-chart-eval="belebele"] .language-count').textContent`),'1'); +assert.equal(await evaluate(`document.querySelector('[data-chart-eval="flores200"] .language-count').textContent`),'2'); await click('#clear');await change('#compareSort','descending'); await change('#compareGroup','variant'); assert.equal(await evaluate("document.querySelectorAll('.comparison-row').length"),usedRows); @@ -180,7 +180,7 @@ assert.equal(await evaluate("document.querySelectorAll('.catalogue-task').length assert.ok((await evaluate("document.querySelector('#filterStatus').textContent")).startsWith(`${new Set(payload.rows.map(r=>r.task)).size} of ${new Set(payload.rows.map(r=>r.task)).size} task names · ${payload.rows.length} metric rows`)); assert.ok(await evaluate("document.querySelectorAll('#view *').length<2500"),'collapsed catalogue rendered too much content'); assert.equal(await evaluate("document.querySelectorAll('.catalogue-metric').length"),0); -assert.match(await evaluate("document.querySelector('[data-eval=\"ARC Challenge\"] > summary .scoring-options').textContent"),/Selected: acc_norm.*Other available metrics: acc/); +assert.match(await evaluate("document.querySelector('[data-eval=\"arc_challenge\"] > summary .scoring-options').textContent"),/Selected: acc_norm.*Other available metrics: acc/); const humanSummary=await evaluate("document.querySelector('[data-eval=HumanEval] > summary').textContent"); assert.match(humanSummary,/Selected: python_pass@1.*0-shot · no examples.*Other available metrics: sh_pass@1/); @@ -203,7 +203,7 @@ assert.match(await evaluate("document.querySelector('#view').textContent"),/Excl await click('#clear'); // Config import changes metric/category/normalization/languages atomically. await evaluate(`window.loadTestConfig=async config=>{for(const [selector,object] of [['#weightsFile',Object.fromEntries(Object.entries(config).filter(([k])=>['version','name','weights','english_weights','aggregate'].includes(k)))],['#configFile',Object.fromEntries(Object.entries(config).filter(([k])=>!['weights','english_weights','aggregate'].includes(k)))]]){const dt=new DataTransfer();dt.items.add(new File([jsyaml.dump(object,{schema:jsyaml.CORE_SCHEMA})],'config.yaml'));const e=document.querySelector(selector);e.files=dt.files;await document.querySelector('#view').onchange({target:e});if(document.querySelector('#error').textContent)break;}};`); -await evaluate(`(async()=>{const c=structuredClone(DATA.scheme);c.name='Browser custom config';const he=c.evals.find(e=>e.name==='HumanEval');he.metric='sh_pass@1';he.category='Math';c.evals.find(e=>e.name==='HellaSwag').normalize={min:.25,max:1};const group=c.languages.find(g=>g.tasks.includes('AIME24'));group.tasks=group.tasks.filter(t=>t!=='AIME24');c.languages=c.languages.filter(g=>g.tasks.length);c.languages.push({tasks:['AIME24'],scope:'single',language:'fin_Latn'});await loadTestConfig(c);})()`); +await evaluate(`(async()=>{const c=structuredClone(DATA.scheme);c.name='Browser custom config';const he=c.evals.find(e=>e.name==='HumanEval');he.metric='sh_pass@1';he.category='Math';c.evals.find(e=>e.name==='hellaswag').normalize={min:.25,max:1};const group=c.languages.find(g=>g.tasks.includes('AIME24'));group.tasks=group.tasks.filter(t=>t!=='AIME24');c.languages=c.languages.filter(g=>g.tasks.length);c.languages.push({tasks:['AIME24'],scope:'single',language:'fin_Latn'});await loadTestConfig(c);})()`); assert.equal(await evaluate("document.querySelector('#error').textContent"),''); assert.notEqual(await evaluate("document.querySelector('#cards strong').textContent"),initialScore); assert.match(await evaluate("document.querySelector('[data-eval=HumanEval] > summary').textContent"),/Math.*sh_pass@1/); @@ -224,20 +224,20 @@ assert.equal(await evaluate("document.querySelector('#error').textContent"),''); await evaluate(`(async()=>{const rows=DATA.rows.filter(r=>!(r.task==='arc_challenge_mt_cs'&&r.metric==='acc_norm')).map(r=>({...r,checkpoint:'Missing Czech metric'}));const fields=Object.keys(rows[0]);const csv=[fields,...rows.map(r=>fields.map(f=>r[f]))].map(row=>row.map(v=>JSON.stringify(String(v??''))).join(',')).join('\\n');const dt=new DataTransfer();dt.items.add(new File([csv],'missing.csv'));const e=document.querySelector('#modelFile');e.files=dt.files;await e.onchange({target:e});})()`); assert.equal(await evaluate("document.querySelector('#error').textContent"),''); assert.equal(await evaluate("document.querySelector('#warningCount').textContent"),'11'); -assert.match(await evaluate("document.querySelector('[data-eval=\"ARC Challenge\"] > summary').textContent"),/1 missing scoring/); -await click('[data-eval="ARC Challenge"] > summary'); +assert.match(await evaluate("document.querySelector('[data-eval=\"arc_challenge\"] > summary').textContent"),/1 missing scoring/); +await click('[data-eval="arc_challenge"] > summary'); await click('[data-task=arc_challenge_mt_cs] > summary'); assert.match(await evaluate("document.querySelector('.has-missing-field').textContent"),/arc_challenge_mt_cs.*Missing Czech metric.*expected acc_norm/s); assert.notEqual(await evaluate("document.querySelector('#cards strong').textContent"),'—'); assert.match(await evaluate("document.querySelector('#coverage').textContent"),/excluded: 1 from A, 0 from B/); await click('[data-view=warnings]'); assert.match(await evaluate("document.querySelector('#view').textContent"),/Missing scoring field.*arc_challenge_mt_cs/s); -assert.match(await evaluate("document.querySelector('#view').textContent"),/Comparison coverage.*ARC Challenge/s); -await evaluate(`(async()=>{const fields=['checkpoint','task','metric','filter','n_shot','harness','backend','value'];const rows=DATA.rows.filter(r=>r.eval!=='IFEval').map(r=>({...r,checkpoint:'Missing IFEval'}));const csv=[fields.join(','),...rows.map(r=>fields.map(f=>r[f]).join(','))].join('\\n');const dt=new DataTransfer();dt.items.add(new File([csv],'missing-eval.csv'));const e=document.querySelector('#modelFile');e.files=dt.files;await e.onchange({target:e});})()`); +assert.match(await evaluate("document.querySelector('#view').textContent"),/Comparison coverage.*arc_challenge/s); +await evaluate(`(async()=>{const fields=['checkpoint','task','metric','filter','n_shot','harness','backend','value'];const rows=DATA.rows.filter(r=>r.eval!=='ifeval').map(r=>({...r,checkpoint:'Missing ifeval'}));const csv=[fields.join(','),...rows.map(r=>fields.map(f=>r[f]).join(','))].join('\\n');const dt=new DataTransfer();dt.items.add(new File([csv],'missing-eval.csv'));const e=document.querySelector('#modelFile');e.files=dt.files;await e.onchange({target:e});})()`); assert.notEqual(await evaluate("document.querySelector('#cards strong').textContent"),'—'); assert.equal(await evaluate("document.querySelector('#cards .score-card:last-child strong').textContent"),'0.00'); assert.doesNotMatch(await evaluate("document.querySelector('#view').textContent"),/No eval data/); -assert.match(await evaluate("document.querySelector('#view').textContent"),/Comparison coverage.*IFEval/s); +assert.match(await evaluate("document.querySelector('#view').textContent"),/Comparison coverage.*ifeval/s); for(const mode of ['standard','english_eval','english_category']){await click('[data-aggregate='+mode+']');assert.equal(await evaluate("document.querySelector('#cards .score-card:last-child strong').textContent"),'0.00');} // Reload restores the original export before the remaining import checks. await send('Page.reload'); @@ -279,40 +279,40 @@ assert.equal(await evaluate("document.querySelector('#warningCount').textContent assert.equal(await evaluate("document.querySelector('#warningCount').classList.contains('has-warnings')"),true); // Warnings expose concrete settings and editable config caveats. await click('[data-view=warnings]'); -assert.match(await evaluate("document.querySelector('#view').textContent"),/Inconsistent scoring settings.*MGSM.*0 shots.*5 shots/s); -assert.match(await evaluate("document.querySelector('#view').textContent"),/Not used.*Global PIQA \(prompted\)/s); +assert.match(await evaluate("document.querySelector('#view').textContent"),/Inconsistent scoring settings.*mgsm.*0 shots.*5 shots/s); +assert.match(await evaluate("document.querySelector('#view').textContent"),/Not used.*global_piqa_prompted/s); await click('[data-view=config]'); -assert.match(await evaluate(`document.querySelector('[data-eval="ARC Challenge"] > summary').textContent`),/Inconsistent scoring settings/); -assert.doesNotMatch(await evaluate(`document.querySelector('[data-eval="SIB-200"] .normalization-info').textContent`),/Metric-selection exception/); +assert.match(await evaluate(`document.querySelector('[data-eval="arc_challenge"] > summary').textContent`),/Inconsistent scoring settings/); +assert.doesNotMatch(await evaluate(`document.querySelector('[data-eval="sib200"] .normalization-info').textContent`),/Metric-selection exception/); assert.match(await evaluate(`document.querySelector('[data-eval="JEEBench"] > summary').textContent`),/10.55% baseline/); assert.match(await evaluate(`document.querySelector('[data-eval="JEEBench"] .normalization-info').textContent`),/Table 2/); -assert.match(await evaluate(`document.querySelector('[data-eval="FLORES200"] .normalization-info').textContent`),/0–100 points/g); -assert.match(await evaluate(`document.querySelector('[data-eval="OpenSubtitles"] .normalization-info').textContent`),/0–100 points/g); -await click('[data-eval="Language ID"] > summary'); +assert.match(await evaluate(`document.querySelector('[data-eval="flores200"] .normalization-info').textContent`),/0–100 points/g); +assert.match(await evaluate(`document.querySelector('[data-eval="opensubtitles_multi40"] .normalization-info').textContent`),/0–100 points/g); +await click('[data-eval="bigbench_language_identification"] > summary'); await click('[data-task="bigbench_language_identification_multiple_choice"] > summary'); assert.match(await evaluate(`document.querySelector('[data-task="bigbench_language_identification_multiple_choice"] .task-details').textContent`),/English-balance group: English \(fallback/); assert.match(await evaluate(`document.querySelector('[data-task="bigbench_language_identification_multiple_choice"]').closest('tr').textContent`),/Multilingual \(pooled\)/); -await click('[data-eval="MultiBlimp"] > summary'); +await click('[data-eval="multiblimp"] > summary'); await click('[data-task="multiblimp_hbs"] > summary'); assert.match(await evaluate(`document.querySelector('[data-task="multiblimp_hbs"] .task-details').textContent`),/English-balance group: Other languages/); assert.match(await evaluate(`document.querySelector('[data-task="multiblimp_hbs"]').closest('tr').textContent`),/srp_Latn/); -assert.match(await evaluate(`document.querySelector('[data-eval="MultiBlimp"] .normalization-info').textContent`),/Language-grouping approximation.*more speakers/); +assert.match(await evaluate(`document.querySelector('[data-eval="multiblimp"] .normalization-info').textContent`),/Language-grouping approximation.*more speakers/); -await click('[data-eval="Language ID"] > summary'); -await click('[data-eval="MultiBlimp"] > summary'); +await click('[data-eval="bigbench_language_identification"] > summary'); +await click('[data-eval="multiblimp"] > summary'); for(const name of ['AIME24','AIME25'])assert.match(await evaluate(`document.querySelector('[data-eval="${name}"] .normalization-info').textContent`),/random_score = 0/); // Custom caveats survive import/export and can be removed when resolved. -await evaluate(`(async()=>{const c=structuredClone(DATA.scheme);delete c.evals.find(e=>e.name==='MultiBlimp').warning;await loadTestConfig(c);})()`); +await evaluate(`(async()=>{const c=structuredClone(DATA.scheme);delete c.evals.find(e=>e.name==='multiblimp').warning;await loadTestConfig(c);})()`); assert.equal(await evaluate("document.querySelector('#warningCount').textContent"),'4'); await evaluate(`loadTestConfig(DATA.scheme)`); // Normalization is visible without opening individual language variants. -assert.match(await evaluate(`document.querySelector('[data-eval="SIB-200"] > summary').textContent`),/14.29% baseline/); -await click('[data-eval="SIB-200"] > summary'); -assert.match(await evaluate(`document.querySelector('[data-eval="SIB-200"] .normalization-info').textContent`),/seven topic choices/); -assert.ok(await evaluate(`document.querySelector('[data-eval="SIB-200"] .normalization-info a').href.startsWith('https://github.com/')`)); -assert.match(await evaluate(`document.querySelector('[data-eval="ARC Challenge"] > summary').textContent`),/25.00% baseline/); -await click('[data-eval="SIB-200"] > summary'); +assert.match(await evaluate(`document.querySelector('[data-eval="sib200"] > summary').textContent`),/14.29% baseline/); +await click('[data-eval="sib200"] > summary'); +assert.match(await evaluate(`document.querySelector('[data-eval="sib200"] .normalization-info').textContent`),/seven topic choices/); +assert.ok(await evaluate(`document.querySelector('[data-eval="sib200"] .normalization-info a').href.startsWith('https://github.com/')`)); +assert.match(await evaluate(`document.querySelector('[data-eval="arc_challenge"] > summary').textContent`),/25.00% baseline/); +await click('[data-eval="sib200"] > summary'); // Header and cell alignment must agree, including the catalogue grid. assert.ok(await evaluate(`(()=>{const headers=[...document.querySelector('.catalogue-head').children],cells=[...document.querySelector('.catalogue-summary').children];return headers.every((h,i)=>Math.abs(h.getBoundingClientRect().x-cells[i].getBoundingClientRect().x)<1)})()`)); await evaluate("document.querySelector('#view').scrollIntoView()");await screenshot('config-preview'); diff --git a/tests/test_check.py b/tests/test_check.py new file mode 100644 index 0000000..ab660ca --- /dev/null +++ b/tests/test_check.py @@ -0,0 +1,39 @@ +"""The pre-PR command must propagate browser failures and clean up Chrome.""" +from pathlib import Path +import subprocess +import unittest +from unittest.mock import Mock, patch +from tests import check + + +class CheckCommand(unittest.TestCase): + def test_browser_uses_own_port_and_cleans_up_on_success_or_failure(self): + for fails in (False, True): + with self.subTest(fails=fails): + browser = Mock() + directories = [] + + def launch(args, **kwargs): + profile = Path(next(a.split('=', 1)[1] for a in args if a.startswith('--user-data-dir='))) + profile.mkdir() + (profile / 'DevToolsActivePort').write_text('43210\n/devtools/browser/test\n') + directories.append(profile.parent) + return browser + + failure = subprocess.CalledProcessError(1, 'node') if fails else None + with patch.object(check.subprocess, 'Popen', side_effect=launch), patch.object(check.subprocess, 'run', side_effect=failure) as run: + if fails: + with self.assertRaises(subprocess.CalledProcessError): + check.run_browser('chrome') + else: + check.run_browser('chrome') + self.assertEqual(run.call_args.kwargs['env']['QUICKDASH_CHROME_PORT'], '43210') + self.assertTrue(run.call_args.kwargs['check']) + browser.terminate.assert_called_once() + browser.wait.assert_called_once_with(timeout=10) + self.assertFalse(directories[0].exists()) + + def test_missing_browser_is_an_error_not_a_skipped_check(self): + with patch.object(check.os.path, 'isfile', return_value=False): + with self.assertRaisesRegex(RuntimeError, 'Chrome is required'): + check.find_chrome() diff --git a/tests/test_engines.py b/tests/test_engines.py index 55de7b4..961534a 100644 --- a/tests/test_engines.py +++ b/tests/test_engines.py @@ -5,6 +5,7 @@ import json import math import random +import re import subprocess import unittest import warnings @@ -149,6 +150,18 @@ def both(self, cases): results.append((py, other)) return results + def test_documented_eval_sets_resolve_in_both_engines(self): + section = (ROOT / "docs/configuration.md").read_text().split("## Strict and relaxed matching")[0] + examples = [parse_yaml(block) for block in re.findall(r"```yaml\n(.*?)```", section, re.S)] + self.assertTrue(examples, "The eval-set guide must contain executable examples") + base = load_config(catalogue=ROOT / "configs/catalogue.yaml", + weights=ROOT / "configs/weights/oellm.yaml") + rows = parse_csv((ROOT / "examples/sample-evals.csv").read_text()) + cases = [dict(config={**base, "suite": suite}, rows=rows) for suite in examples] + for results in self.both(cases): + for result in results: + self.assertNotIn("error", result) + def test_per_eval_catalogue_assembly(self): original = fixture() metadata = {"version": 1, "name": "Fixture", "notes": ["Shared catalogue"]} diff --git a/tests/test_public_browser.mjs b/tests/test_public_browser.mjs index f1075c3..22e8bda 100644 --- a/tests/test_public_browser.mjs +++ b/tests/test_public_browser.mjs @@ -25,10 +25,10 @@ try{ // A cold CI runner can take longer than five seconds to launch Chrome. const startupDeadline=Date.now()+30_000; while(Date.now()t.type==='page'))break;}catch{} + try{tabs=await(await fetch(`http://127.0.0.1:${process.env.QUICKDASH_CHROME_PORT || 9227}/json/list`)).json();if(tabs.some(t=>t.type==='page'))break;}catch{} await new Promise(r=>setTimeout(r,200)); } - assert.ok(tabs?.some(t=>t.type==='page'),'Start an isolated Chrome session on port 9227'); + assert.ok(tabs?.some(t=>t.type==='page'),'Start an isolated Chrome session on QUICKDASH_CHROME_PORT (default 9227)'); ws=new WebSocket(tabs.find(t=>t.type==='page').webSocketDebuggerUrl); await new Promise(r=>ws.addEventListener('open',r,{once:true})); let id=0;const pending=new Map(),errors=[],network=[]; @@ -262,15 +262,15 @@ try{ const strictSampleCards=await evaluate("document.querySelector('#cards').textContent"); // Strict shot mismatches collapse per eval, including duplicate missing-set warnings. await change('#suitePreset',await evaluate("String(DATA.suites.findIndex(s=>s.file==='flagship-1.yaml'))"));await click('[data-view=warnings]'); - const arcWarnings=await evaluate("[...document.querySelectorAll('#view tbody tr')].filter(r=>r.cells[1]?.textContent==='ARC Challenge').map(r=>r.textContent)"); + const arcWarnings=await evaluate("[...document.querySelectorAll('#view tbody tr')].filter(r=>r.cells[1]?.textContent==='arc_challenge').map(r=>r.textContent)"); assert.equal(arcWarnings.length,1); assert.match(arcWarnings[0],/Few-shot mismatch excluded.*\d+ tasks use 0 shots; expected 10/s); assert.match(arcWarnings[0],/arc_challenge_mt_cs/); assert.match(await evaluate("document.querySelector('#coverage').textContent"),/INCOMPLETE/); await click('[data-view=config]'); - await evaluate("document.querySelector('.catalogue-eval[data-eval=\"ARC Challenge\"]').open=true"); + await evaluate("document.querySelector('.catalogue-eval[data-eval=\"arc_challenge\"]').open=true"); await new Promise(r=>setTimeout(r,50)); - assert.ok(await evaluate("document.querySelectorAll('.catalogue-eval[data-eval=\"ARC Challenge\"] .has-missing-field').length>0")); + assert.ok(await evaluate("document.querySelectorAll('.catalogue-eval[data-eval=\"arc_challenge\"] .has-missing-field').length>0")); await change('#suitePreset',await evaluate("String(DATA.suites.findIndex(s=>s.file==='any-available.yaml'))")); assert.equal(await evaluate("document.querySelector('#cards').textContent"),strictSampleCards);