diff --git a/analysis/export_dashboard.py b/analysis/export_dashboard.py
index a6d0bb1..2206499 100644
--- a/analysis/export_dashboard.py
+++ b/analysis/export_dashboard.py
@@ -17,6 +17,7 @@
import pandas as pd
from analysis.constants import (
+ COMMONCRAWL_STATS_CSV,
CONTRIBUTOR_COUNTS_CSV,
DERIVED_DIR,
DIFFICULTY_OUTPUT_DIR,
@@ -46,6 +47,14 @@
from analysis.tables.model_performance_summary import build_model_performance
OUT = DERIVED_DIR / "dashboard" / "dashboard_data.json"
+# Language-resource levels by Common Crawl page count, as defined in the paper.
+# A level covers counts from min_pages (inclusive) up to max_pages (exclusive).
+RESOURCE_LEVELS = (
+ {"group": "high", "min_pages": 10_000_000, "max_pages": None},
+ {"group": "medium", "min_pages": 100_000, "max_pages": 10_000_000},
+ {"group": "low", "min_pages": 10_000, "max_pages": 100_000},
+ {"group": "extremely-low", "min_pages": None, "max_pages": 10_000},
+)
PAIR_MODELS = {
"claude_reasoning": ("claude-opus-4-7", "claude-opus-4-7-thinking"),
"deepseek_reasoning": ("deepseek-v4-pro", "deepseek-v4-pro-thinking"),
@@ -213,7 +222,22 @@ def export() -> dict:
on="community",
validate="1:1",
).sort_values("mean_accuracy")
+ crawl = pd.read_csv(COMMONCRAWL_STATS_CSV)
+ pages = meta[["community", "language_code"]].merge(
+ crawl[["language_code", "common_crawl_pages", "mapping_note"]],
+ on="language_code",
+ how="left",
+ validate="m:1",
+ )
+ communities = communities.merge(
+ pages.drop(columns="language_code").rename(columns={"mapping_note": "common_crawl_note"}),
+ on="community",
+ how="left",
+ validate="1:1",
+ )
out["communities"] = records(communities)
+ out["resource_levels"] = list(RESOURCE_LEVELS)
+ out["common_crawl_archive"] = sorted(crawl["common_crawl_archive"].dropna().unique())
out["cells"] = records(cells[["model_display", "community", "value", "total"]])
grouping = community_accuracy()
diff --git a/dashboard/compute.py b/dashboard/compute.py
index 1fc6460..1a5d64f 100644
--- a/dashboard/compute.py
+++ b/dashboard/compute.py
@@ -307,6 +307,20 @@ def indexed(name, rows, key):
communities[community]["pooled_accuracy"],
)
+ levels = indexed("resource levels", d["resource_levels"], "group")
+ check("resource levels match groups", set(levels), {x["group"] for x in d["resource"]})
+ for community, row in communities.items():
+ pages = row.get("common_crawl_pages")
+ if pages is None:
+ continue
+ level = levels[row["language_resourcedness"]]
+ low, high = level["min_pages"], level["max_pages"]
+ check(
+ f"{community} Common Crawl pages within its level",
+ (low is None or pages >= low) and (high is None or pages < high),
+ True,
+ )
+
matrix = d["riddle_matrix"]
solved_by = matrix["solved_by"]
check("riddle matrix models", matrix["models"], [row["model"] for row in d["leaderboard"]])
diff --git a/dashboard/template.html b/dashboard/template.html
index bab99b4..7213eab 100644
--- a/dashboard/template.html
+++ b/dashboard/template.html
@@ -322,7 +322,7 @@
Language resources
-
Each dot is a community, placed by the accuracy of all its scored answers pooled over models. Diamonds pool every answer in the level, with 95% intervals from resampling communities. Levels follow Common Crawl page counts for the language.
+
Each dot is a community, placed by the accuracy of all its scored answers pooled over models. Diamonds pool every answer in the level, with 95% intervals from resampling communities.
Source: Section 5 and Figure 8 of the paper.
@@ -528,6 +528,7 @@
All model configurations
region: c.un_m49_subregion, resource: c.language_resourcedness, writers: c.riddle_writers,
mean: c.mean_accuracy, median_model: c.median_model, frontier: c.frontier_accuracy, small: c.small_accuracy,
below10: c.n_below_10, above20: c.n_above_20, pooled: c.pooled_accuracy, rank_from_hardest: i + 1,
+ pages: c.common_crawl_pages, pages_note: c.common_crawl_note,
}));
const ci = Object.fromEntries(comms.map((c, i) => [c.id, i]));
const cell = Object.fromEntries(R.cells.map(x => [x.model_display + '|' + x.community, x]));
@@ -555,6 +556,7 @@
All model configurations
pairs: R.pairs, pair_models: R.pair_models, search_losses: R.search_losses, english: R.english, english_models: R.english_models,
english_high: R.english_high_resource_accuracy, chinese: R.chinese, tier_effects: R.tier_effects, local_term: R.local_term_effect,
coverage: R.coverage, frontier_models: R.frontier_models, small_models: R.small_models, links: R.links || {},
+ levels: R.resource_levels, cc_archive: (R.common_crawl_archive || []).join(', '),
};
}
const $ = s => document.querySelector(s);
@@ -571,6 +573,16 @@
All model configurations
const label = id => (byId[id] || {}).label || id;
const MOBILE = () => window.innerWidth < 640;
const MD = D.md;
+const pagesFmt = n => n >= 1e6 ? (n / 1e6).toLocaleString('en-US', {maximumFractionDigits: 1}) + 'M' : n >= 1e3 ? Math.round(n / 1e3).toLocaleString('en-US') + 'K' : n0(n);
+function levelRange(l) {
+ if (l.min_pages == null) return `under ${pagesFmt(l.max_pages)} pages, or not detected`;
+ if (l.max_pages == null) return `${pagesFmt(l.min_pages)} pages or more`;
+ return `${pagesFmt(l.min_pages)} to ${pagesFmt(l.max_pages)} pages`;
+}
+const RES_DEF = `Resource levels count the web pages in the language in Common Crawl${D.cc_archive ? ' (' + D.cc_archive + ')' : ''}: ` +
+ D.levels.map(l => `${RES_LABEL[l.group].toLowerCase()}, ${levelRange(l)}`).join('; ') +
+ `. Dialects take their main language's count; for example, every Arabic variety uses the count for Arabic.`;
+document.getElementById('hm-res-btn').title = RES_DEF;
document.querySelectorAll('[data-k="nhe"]').forEach(e => e.textContent = C.length);
document.querySelectorAll('[data-k="nm"]').forEach(e => e.textContent = M.length);
// Communities where no model reaches a quarter of the riddles.
@@ -865,7 +877,8 @@
All model configurations
const c = byId[cur], j = c.i;
const easyRank = C.length - c.rank_from_hardest + 1;
$('#ex-name').textContent = c.label;
- $('#ex-meta').innerHTML = `${c.language} · ${c.country} · ${c.region}
${RES_LABEL[c.resource]} resource`;
+ const pagesText = c.pages == null ? '' : c.pages > 0 ? ` · about ${pagesFmt(c.pages)} Common Crawl pages` : ' · not detected in Common Crawl';
+ $('#ex-meta').innerHTML = `${c.language} · ${c.country} · ${c.region}
${RES_LABEL[c.resource]} resource${pagesText}`;
$('#ex-link').href = '#community=' + cur;
const vals = M.map((m, i) => D.grid[i][j]);
const bi = vals.indexOf(Math.max(...vals));
@@ -908,6 +921,7 @@
All model configurations
/* ---------- resource ---------- */
register(function resource() {
+ $('#res-def').textContent = RES_DEF;
const tiers = ['high', 'medium', 'low', 'extremely-low'];
const R = Object.fromEntries(D.resource.map(r => [r.group, r]));
const tr = [];
@@ -919,7 +933,7 @@
All model configurations
hovertemplate: '
%{text}%{x:.1f}%
'});
tr.push({type: 'scatter', mode: 'markers', x: [R[t].pooled], y: [k], error_x: {type: 'data', symmetric: false, array: [R[t].chi - R[t].pooled], arrayminus: [R[t].pooled - R[t].clo], color: css('--ink'), thickness: 2, width: 6},
marker: {symbol: 'diamond', size: 18, color: css('--ink'), line: {width: 2, color: css('--card')}},
- hovertemplate: `
${RES_LABEL[t]} resourcepooled %{x:.1f}%
${R[t].communities} communities
`});
+ hovertemplate: `
${RES_LABEL[t]} resource (${levelRange(D.levels.find(l => l.group === t))})
pooled %{x:.1f}%
${R[t].communities} communities
`});
});
plot('resource', tr, base({height: 380, margin: {l: 10, r: 20, t: 6, b: 44},
xaxis: {range: [0, 80], ticksuffix: '%', gridcolor: css('--grid'), fixedrange: true, title: {text: 'accuracy', font: {size: 12}}},
diff --git a/data/derived/dashboard/dashboard_data.json b/data/derived/dashboard/dashboard_data.json
index 3111495..b8cb17c 100644
--- a/data/derived/dashboard/dashboard_data.json
+++ b/data/derived/dashboard/dashboard_data.json
@@ -1683,7 +1683,9 @@
"country_code": "IDN",
"language_resourcedness": "extremely-low",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 0.0,
+ "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)."
},
{
"community": "bugisnese_indonesia",
@@ -1700,7 +1702,9 @@
"country_code": "IDN",
"language_resourcedness": "extremely-low",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 0.0,
+ "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)."
},
{
"community": "lumasaba_uganda",
@@ -1717,7 +1721,9 @@
"country_code": "UGA",
"language_resourcedness": "extremely-low",
"un_m49_subregion": "Sub-Saharan Africa",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 0.0,
+ "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)."
},
{
"community": "arabic_sudan",
@@ -1734,7 +1740,9 @@
"country_code": "SDN",
"language_resourcedness": "high",
"un_m49_subregion": "North Africa",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 14286482.0,
+ "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2."
},
{
"community": "yoruba_nigeria",
@@ -1751,7 +1759,9 @@
"country_code": "NGA",
"language_resourcedness": "low",
"un_m49_subregion": "Sub-Saharan Africa",
- "riddle_writers": 4
+ "riddle_writers": 4,
+ "common_crawl_pages": 36052.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "luganda_uganda",
@@ -1768,7 +1778,9 @@
"country_code": "UGA",
"language_resourcedness": "extremely-low",
"un_m49_subregion": "Sub-Saharan Africa",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 7704.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "afaan_oromo_oromo",
@@ -1785,7 +1797,9 @@
"country_code": "ETH",
"language_resourcedness": "low",
"un_m49_subregion": "Sub-Saharan Africa",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": null,
+ "common_crawl_note": null
},
{
"community": "swahili_kenya",
@@ -1802,7 +1816,9 @@
"country_code": "KEN",
"language_resourcedness": "medium",
"un_m49_subregion": "Sub-Saharan Africa",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 249945.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "assamese_india",
@@ -1819,7 +1835,9 @@
"country_code": "IND",
"language_resourcedness": "low",
"un_m49_subregion": "South Asia",
- "riddle_writers": 4
+ "riddle_writers": 4,
+ "common_crawl_pages": 66726.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "belarusian_belarus",
@@ -1836,7 +1854,9 @@
"country_code": "BLR",
"language_resourcedness": "medium",
"un_m49_subregion": "Eastern Europe",
- "riddle_writers": 4
+ "riddle_writers": 4,
+ "common_crawl_pages": 367299.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "kannada_india",
@@ -1853,7 +1873,9 @@
"country_code": "IND",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 323079.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "malayalam_india",
@@ -1870,7 +1892,9 @@
"country_code": "IND",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 509416.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "konkani_india",
@@ -1887,7 +1911,9 @@
"country_code": "IND",
"language_resourcedness": "extremely-low",
"un_m49_subregion": "South Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 0.0,
+ "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)."
},
{
"community": "hungarian_hungary",
@@ -1904,7 +1930,9 @@
"country_code": "HUN",
"language_resourcedness": "high",
"un_m49_subregion": "Eastern Europe",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 11427802.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "chinese_malaysia",
@@ -1921,7 +1949,9 @@
"country_code": "MYS",
"language_resourcedness": "high",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 93776922.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "uzbek_uzbekistan",
@@ -1938,7 +1968,9 @@
"country_code": "UZB",
"language_resourcedness": "medium",
"un_m49_subregion": "Central Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 669583.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "german_germany",
@@ -1955,7 +1987,9 @@
"country_code": "DEU",
"language_resourcedness": "high",
"un_m49_subregion": "Western Europe",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 126414617.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "arabic_syria",
@@ -1972,7 +2006,9 @@
"country_code": "SYR",
"language_resourcedness": "high",
"un_m49_subregion": "West Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 14286482.0,
+ "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2."
},
{
"community": "persian_iran",
@@ -1989,7 +2025,9 @@
"country_code": "IRN",
"language_resourcedness": "high",
"un_m49_subregion": "South Asia",
- "riddle_writers": 10
+ "riddle_writers": 10,
+ "common_crawl_pages": 16267415.0,
+ "common_crawl_note": "Mapped from Iranian Persian (pes) to Persian macro-language (fas)."
},
{
"community": "thai_thailand",
@@ -2006,7 +2044,9 @@
"country_code": "THA",
"language_resourcedness": "medium",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 5
+ "riddle_writers": 5,
+ "common_crawl_pages": 8002604.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "chinese_china",
@@ -2023,7 +2063,9 @@
"country_code": "CHN",
"language_resourcedness": "high",
"un_m49_subregion": "East Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 93776922.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "spanish_argentina",
@@ -2040,7 +2082,9 @@
"country_code": "ARG",
"language_resourcedness": "high",
"un_m49_subregion": "Latin America",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 98972317.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "cantonese_china",
@@ -2057,7 +2101,9 @@
"country_code": "CHN",
"language_resourcedness": "extremely-low",
"un_m49_subregion": "East Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 0.0,
+ "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)."
},
{
"community": "telugu_india",
@@ -2074,7 +2120,9 @@
"country_code": "IND",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 8
+ "riddle_writers": 8,
+ "common_crawl_pages": 425530.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "catalan_catalonia",
@@ -2091,7 +2139,9 @@
"country_code": "ESP",
"language_resourcedness": "medium",
"un_m49_subregion": "Southern Europe",
- "riddle_writers": 5
+ "riddle_writers": 5,
+ "common_crawl_pages": 4383390.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "nepali_nepal",
@@ -2108,7 +2158,9 @@
"country_code": "NPL",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 1274697.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "chinese_simplified_china",
@@ -2125,7 +2177,9 @@
"country_code": "CHN",
"language_resourcedness": "high",
"un_m49_subregion": "East Asia",
- "riddle_writers": 7
+ "riddle_writers": 7,
+ "common_crawl_pages": 93776922.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "luxembourgish_luxembourg",
@@ -2142,7 +2196,9 @@
"country_code": "LUX",
"language_resourcedness": "low",
"un_m49_subregion": "Western Europe",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 80957.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "russian_russia",
@@ -2159,7 +2215,9 @@
"country_code": "RUS",
"language_resourcedness": "high",
"un_m49_subregion": "Eastern Europe",
- "riddle_writers": 5
+ "riddle_writers": 5,
+ "common_crawl_pages": 146902975.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "indonesian_indonesia",
@@ -2176,7 +2234,9 @@
"country_code": "IDN",
"language_resourcedness": "high",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 23924783.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "ukrainian_ukraine",
@@ -2193,7 +2253,9 @@
"country_code": "UKR",
"language_resourcedness": "high",
"un_m49_subregion": "Eastern Europe",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 17911708.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "serbian_serbia",
@@ -2210,7 +2272,9 @@
"country_code": "SRB",
"language_resourcedness": "medium",
"un_m49_subregion": "Southern Europe",
- "riddle_writers": 4
+ "riddle_writers": 4,
+ "common_crawl_pages": 5713950.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "polish_poland",
@@ -2227,7 +2291,9 @@
"country_code": "POL",
"language_resourcedness": "high",
"un_m49_subregion": "Eastern Europe",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 44157233.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "urdu_pakistan",
@@ -2244,7 +2310,9 @@
"country_code": "PAK",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 689187.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "arabic_morocco",
@@ -2261,7 +2329,9 @@
"country_code": "MAR",
"language_resourcedness": "high",
"un_m49_subregion": "North Africa",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 14286482.0,
+ "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2."
},
{
"community": "tamil_india",
@@ -2278,7 +2348,9 @@
"country_code": "IND",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 960601.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "marathi_india",
@@ -2295,7 +2367,9 @@
"country_code": "IND",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 571814.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "arabic_egypt",
@@ -2312,7 +2386,9 @@
"country_code": "EGY",
"language_resourcedness": "high",
"un_m49_subregion": "North Africa",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 14286482.0,
+ "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2."
},
{
"community": "bengali_bangladesh",
@@ -2329,7 +2405,9 @@
"country_code": "BGD",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 2318326.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "bulgarian_bulgaria",
@@ -2346,7 +2424,9 @@
"country_code": "BGR",
"language_resourcedness": "medium",
"un_m49_subregion": "Eastern Europe",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 6211572.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "italian_italy",
@@ -2363,7 +2443,9 @@
"country_code": "ITA",
"language_resourcedness": "high",
"un_m49_subregion": "Southern Europe",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 52686575.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "tunisian_arabic_tunisia",
@@ -2380,7 +2462,9 @@
"country_code": "TUN",
"language_resourcedness": "high",
"un_m49_subregion": "North Africa",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 14286482.0,
+ "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2."
},
{
"community": "portuguese_portugal",
@@ -2397,7 +2481,9 @@
"country_code": "PRT",
"language_resourcedness": "high",
"un_m49_subregion": "Southern Europe",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 54053402.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "vietnamese_vietnam",
@@ -2414,7 +2500,9 @@
"country_code": "VNM",
"language_resourcedness": "high",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 23000895.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "spanish_peru",
@@ -2431,7 +2519,9 @@
"country_code": "PER",
"language_resourcedness": "high",
"un_m49_subregion": "Latin America",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 98972317.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "sinhala_sri_lanka",
@@ -2448,7 +2538,9 @@
"country_code": "LKA",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 189589.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "bengali_india",
@@ -2465,7 +2557,9 @@
"country_code": "IND",
"language_resourcedness": "medium",
"un_m49_subregion": "South Asia",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 2318326.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "spanish_cuba",
@@ -2482,7 +2576,9 @@
"country_code": "CUB",
"language_resourcedness": "high",
"un_m49_subregion": "Caribbean",
- "riddle_writers": 3
+ "riddle_writers": 3,
+ "common_crawl_pages": 98972317.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "chinese_singapore",
@@ -2499,7 +2595,9 @@
"country_code": "SGP",
"language_resourcedness": "high",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 93776922.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "french_canada",
@@ -2516,7 +2614,9 @@
"country_code": "CAN",
"language_resourcedness": "high",
"un_m49_subregion": "North America",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 103720803.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "malay_malaysia",
@@ -2533,7 +2633,9 @@
"country_code": "MYS",
"language_resourcedness": "medium",
"un_m49_subregion": "Southeast Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 1840421.0,
+ "common_crawl_note": "Mapped from Standard Malay (zsm) to Malay macro-language (msa)."
},
{
"community": "english_canada",
@@ -2550,7 +2652,9 @@
"country_code": "CAN",
"language_resourcedness": "high",
"un_m49_subregion": "North America",
- "riddle_writers": 4
+ "riddle_writers": 4,
+ "common_crawl_pages": 865530140.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "dutch_netherlands",
@@ -2567,7 +2671,9 @@
"country_code": "NLD",
"language_resourcedness": "high",
"un_m49_subregion": "Western Europe",
- "riddle_writers": 4
+ "riddle_writers": 4,
+ "common_crawl_pages": 38004267.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "brazilian_portuguese",
@@ -2584,7 +2690,9 @@
"country_code": "BRA",
"language_resourcedness": "high",
"un_m49_subregion": "Latin America",
- "riddle_writers": 9
+ "riddle_writers": 9,
+ "common_crawl_pages": 54053402.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "french_france",
@@ -2601,7 +2709,9 @@
"country_code": "FRA",
"language_resourcedness": "high",
"un_m49_subregion": "Western Europe",
- "riddle_writers": 2
+ "riddle_writers": 2,
+ "common_crawl_pages": 103720803.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "japanese_japan",
@@ -2618,7 +2728,9 @@
"country_code": "JPN",
"language_resourcedness": "high",
"un_m49_subregion": "East Asia",
- "riddle_writers": 1
+ "riddle_writers": 1,
+ "common_crawl_pages": 112295117.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
},
{
"community": "english_usa",
@@ -2635,9 +2747,36 @@
"country_code": "USA",
"language_resourcedness": "high",
"un_m49_subregion": "North America",
- "riddle_writers": 5
+ "riddle_writers": 5,
+ "common_crawl_pages": 865530140.0,
+ "common_crawl_note": "Exact ISO 639-3 code match."
}
],
+ "resource_levels": [
+ {
+ "group": "high",
+ "min_pages": 10000000,
+ "max_pages": null
+ },
+ {
+ "group": "medium",
+ "min_pages": 100000,
+ "max_pages": 10000000
+ },
+ {
+ "group": "low",
+ "min_pages": 10000,
+ "max_pages": 100000
+ },
+ {
+ "group": "extremely-low",
+ "min_pages": null,
+ "max_pages": 10000
+ }
+ ],
+ "common_crawl_archive": [
+ "CC-MAIN-2026-34"
+ ],
"cells": [
{
"model_display": "Amazon Nova Lite",