Skip to content

Explain resource levels (Common Crawl pages) on the dashboard #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
VinodAnbalagan merged 1 commit into from
Oct 22, 2026
Merged

Explain resource levels (Common Crawl pages) on the dashboard #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions analysis/export_dashboard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import pandas as pd

from analysis.constants import (
COMMONCRAWL_STATS_CSV,
CONTRIBUTOR_COUNTS_CSV,
DERIVED_DIR,
DIFFICULTY_OUTPUT_DIR,
Expand Down Expand Up @@ -46,6 +47,14 @@
from analysis.tables.model_performance_summary import build_model_performance

OUT = DERIVED_DIR / "dashboard" / "dashboard_data.json"
# Language-resource levels by Common Crawl page count, as defined in the paper.
# A level covers counts from min_pages (inclusive) up to max_pages (exclusive).
RESOURCE_LEVELS = (
{"group": "high", "min_pages": 10_000_000, "max_pages": None},
{"group": "medium", "min_pages": 100_000, "max_pages": 10_000_000},
{"group": "low", "min_pages": 10_000, "max_pages": 100_000},
{"group": "extremely-low", "min_pages": None, "max_pages": 10_000},
)
PAIR_MODELS = {
"claude_reasoning": ("claude-opus-4-7", "claude-opus-4-7-thinking"),
"deepseek_reasoning": ("deepseek-v4-pro", "deepseek-v4-pro-thinking"),
Expand Down Expand Up @@ -213,7 +222,22 @@ def export() -> dict:
on="community",
validate="1:1",
).sort_values("mean_accuracy")
crawl = pd.read_csv(COMMONCRAWL_STATS_CSV)
pages = meta[["community", "language_code"]].merge(
crawl[["language_code", "common_crawl_pages", "mapping_note"]],
on="language_code",
how="left",
validate="m:1",
)
communities = communities.merge(
pages.drop(columns="language_code").rename(columns={"mapping_note": "common_crawl_note"}),
on="community",
how="left",
validate="1:1",
)
out["communities"] = records(communities)
out["resource_levels"] = list(RESOURCE_LEVELS)
out["common_crawl_archive"] = sorted(crawl["common_crawl_archive"].dropna().unique())
out["cells"] = records(cells[["model_display", "community", "value", "total"]])

grouping = community_accuracy()
Expand Down
14 changes: 14 additions & 0 deletions dashboard/compute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,20 @@ def indexed(name, rows, key):
communities[community]["pooled_accuracy"],
)

levels = indexed("resource levels", d["resource_levels"], "group")
check("resource levels match groups", set(levels), {x["group"] for x in d["resource"]})
for community, row in communities.items():
pages = row.get("common_crawl_pages")
if pages is None:
continue
level = levels[row["language_resourcedness"]]
low, high = level["min_pages"], level["max_pages"]
check(
f"{community} Common Crawl pages within its level",
(low is None or pages >= low) and (high is None or pages < high),
True,
)

matrix = d["riddle_matrix"]
solved_by = matrix["solved_by"]
check("riddle matrix models", matrix["models"], [row["model"] for row in d["leaderboard"]])
Expand Down
22 changes: 18 additions & 4 deletions dashboard/template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ <h3>Cultural fingerprint</h3>
<div class="seg" id="hm-sort" role="group" aria-label="Order communities">
<button data-s="mean" class="on">By difficulty</button>
<button data-s="region">By region</button>
<button data-s="resource">By language resources</button>
<button data-s="resource" id="hm-res-btn">By language resources</button>
</div>
</div>
<div class="tscroll"><div id="heat" class="plot"></div></div>
Expand Down Expand Up @@ -361,7 +361,7 @@ <h4 style="margin-top:18px">How many models solve each riddle</h4>

<div class="card reveal">
<h3>Language resources</h3>
<p class="sub">Each dot is a community, placed by the accuracy of all its scored answers pooled over models. Diamonds pool every answer in the level, with 95% intervals from resampling communities. Levels follow Common Crawl page counts for the language.</p>
<p class="sub">Each dot is a community, placed by the accuracy of all its scored answers pooled over models. Diamonds pool every answer in the level, with 95% intervals from resampling communities. <span id="res-def"></span></p>
<div id="resource" class="plot"></div>
<p class="take" id="res-take"></p>
<div class="src">Source: Section 5 and Figure 8 of the paper.</div>
Expand Down Expand Up @@ -528,6 +528,7 @@ <h3>All model configurations</h3>
region: c.un_m49_subregion, resource: c.language_resourcedness, writers: c.riddle_writers,
mean: c.mean_accuracy, median_model: c.median_model, frontier: c.frontier_accuracy, small: c.small_accuracy,
below10: c.n_below_10, above20: c.n_above_20, pooled: c.pooled_accuracy, rank_from_hardest: i + 1,
pages: c.common_crawl_pages, pages_note: c.common_crawl_note,
}));
const ci = Object.fromEntries(comms.map((c, i) => [c.id, i]));
const cell = Object.fromEntries(R.cells.map(x => [x.model_display + '|' + x.community, x]));
Expand Down Expand Up @@ -555,6 +556,7 @@ <h3>All model configurations</h3>
pairs: R.pairs, pair_models: R.pair_models, search_losses: R.search_losses, english: R.english, english_models: R.english_models,
english_high: R.english_high_resource_accuracy, chinese: R.chinese, tier_effects: R.tier_effects, local_term: R.local_term_effect,
coverage: R.coverage, frontier_models: R.frontier_models, small_models: R.small_models, links: R.links || {},
levels: R.resource_levels, cc_archive: (R.common_crawl_archive || []).join(', '),
};
}
const $ = s => document.querySelector(s);
Expand All @@ -571,6 +573,16 @@ <h3>All model configurations</h3>
const label = id => (byId[id] || {}).label || id;
const MOBILE = () => window.innerWidth < 640;
const MD = D.md;
const pagesFmt = n => n >= 1e6 ? (n / 1e6).toLocaleString('en-US', {maximumFractionDigits: 1}) + 'M' : n >= 1e3 ? Math.round(n / 1e3).toLocaleString('en-US') + 'K' : n0(n);
function levelRange(l) {
if (l.min_pages == null) return `under ${pagesFmt(l.max_pages)} pages, or not detected`;
if (l.max_pages == null) return `${pagesFmt(l.min_pages)} pages or more`;
return `${pagesFmt(l.min_pages)} to ${pagesFmt(l.max_pages)} pages`;
}
const RES_DEF = `Resource levels count the web pages in the language in Common Crawl${D.cc_archive ? ' (' + D.cc_archive + ')' : ''}: ` +
D.levels.map(l => `${RES_LABEL[l.group].toLowerCase()}, ${levelRange(l)}`).join('; ') +
`. Dialects take their main language's count; for example, every Arabic variety uses the count for Arabic.`;
document.getElementById('hm-res-btn').title = RES_DEF;
document.querySelectorAll('[data-k="nhe"]').forEach(e => e.textContent = C.length);
document.querySelectorAll('[data-k="nm"]').forEach(e => e.textContent = M.length);
// Communities where no model reaches a quarter of the riddles.
Expand Down Expand Up @@ -865,7 +877,8 @@ <h3>All model configurations</h3>
const c = byId[cur], j = c.i;
const easyRank = C.length - c.rank_from_hardest + 1;
$('#ex-name').textContent = c.label;
$('#ex-meta').innerHTML = `${c.language} · ${c.country} · ${c.region}<span class="chip">${RES_LABEL[c.resource]} resource</span>`;
const pagesText = c.pages == null ? '' : c.pages > 0 ? ` · about ${pagesFmt(c.pages)} Common Crawl pages` : ' · not detected in Common Crawl';
$('#ex-meta').innerHTML = `${c.language} · ${c.country} · ${c.region}<span class="chip" title="${RES_DEF}">${RES_LABEL[c.resource]} resource</span><span style="font-size:13.5px">${pagesText}</span>`;
$('#ex-link').href = '#community=' + cur;
const vals = M.map((m, i) => D.grid[i][j]);
const bi = vals.indexOf(Math.max(...vals));
Expand Down Expand Up @@ -908,6 +921,7 @@ <h3>All model configurations</h3>

/* ---------- resource ---------- */
register(function resource() {
$('#res-def').textContent = RES_DEF;
const tiers = ['high', 'medium', 'low', 'extremely-low'];
const R = Object.fromEntries(D.resource.map(r => [r.group, r]));
const tr = [];
Expand All @@ -919,7 +933,7 @@ <h3>All model configurations</h3>
hovertemplate: '<b>%{text}</b><br>%{x:.1f}%<extra></extra>'});
tr.push({type: 'scatter', mode: 'markers', x: [R[t].pooled], y: [k], error_x: {type: 'data', symmetric: false, array: [R[t].chi - R[t].pooled], arrayminus: [R[t].pooled - R[t].clo], color: css('--ink'), thickness: 2, width: 6},
marker: {symbol: 'diamond', size: 18, color: css('--ink'), line: {width: 2, color: css('--card')}},
hovertemplate: `<b>${RES_LABEL[t]} resource</b><br>pooled %{x:.1f}%<br>${R[t].communities} communities<extra></extra>`});
hovertemplate: `<b>${RES_LABEL[t]} resource</b> (${levelRange(D.levels.find(l => l.group === t))})<br>pooled %{x:.1f}%<br>${R[t].communities} communities<extra></extra>`});
});
plot('resource', tr, base({height: 380, margin: {l: 10, r: 20, t: 6, b: 44},
xaxis: {range: [0, 80], ticksuffix: '%', gridcolor: css('--grid'), fixedrange: true, title: {text: 'accuracy', font: {size: 12}}},
Expand Down
Loading
Loading