From 5cc15f816d63cc39f8df6883a6b81f72cadeb389 Mon Sep 17 00:00:00 2001 From: Vinod Date: Wed, 21 Oct 2026 23:36:18 -0400 Subject: [PATCH] Explain resource levels (Common Crawl pages) on the dashboard --- analysis/export_dashboard.py | 24 ++ dashboard/compute.py | 14 ++ dashboard/template.html | 22 +- data/derived/dashboard/dashboard_data.json | 253 ++++++++++++++++----- 4 files changed, 252 insertions(+), 61 deletions(-) diff --git a/analysis/export_dashboard.py b/analysis/export_dashboard.py index a6d0bb1..2206499 100644 --- a/analysis/export_dashboard.py +++ b/analysis/export_dashboard.py @@ -17,6 +17,7 @@ import pandas as pd from analysis.constants import ( + COMMONCRAWL_STATS_CSV, CONTRIBUTOR_COUNTS_CSV, DERIVED_DIR, DIFFICULTY_OUTPUT_DIR, @@ -46,6 +47,14 @@ from analysis.tables.model_performance_summary import build_model_performance OUT = DERIVED_DIR / "dashboard" / "dashboard_data.json" +# Language-resource levels by Common Crawl page count, as defined in the paper. +# A level covers counts from min_pages (inclusive) up to max_pages (exclusive). +RESOURCE_LEVELS = ( + {"group": "high", "min_pages": 10_000_000, "max_pages": None}, + {"group": "medium", "min_pages": 100_000, "max_pages": 10_000_000}, + {"group": "low", "min_pages": 10_000, "max_pages": 100_000}, + {"group": "extremely-low", "min_pages": None, "max_pages": 10_000}, +) PAIR_MODELS = { "claude_reasoning": ("claude-opus-4-7", "claude-opus-4-7-thinking"), "deepseek_reasoning": ("deepseek-v4-pro", "deepseek-v4-pro-thinking"), @@ -213,7 +222,22 @@ def export() -> dict: on="community", validate="1:1", ).sort_values("mean_accuracy") + crawl = pd.read_csv(COMMONCRAWL_STATS_CSV) + pages = meta[["community", "language_code"]].merge( + crawl[["language_code", "common_crawl_pages", "mapping_note"]], + on="language_code", + how="left", + validate="m:1", + ) + communities = communities.merge( + pages.drop(columns="language_code").rename(columns={"mapping_note": "common_crawl_note"}), + on="community", + how="left", + validate="1:1", + ) out["communities"] = records(communities) + out["resource_levels"] = list(RESOURCE_LEVELS) + out["common_crawl_archive"] = sorted(crawl["common_crawl_archive"].dropna().unique()) out["cells"] = records(cells[["model_display", "community", "value", "total"]]) grouping = community_accuracy() diff --git a/dashboard/compute.py b/dashboard/compute.py index 1fc6460..1a5d64f 100644 --- a/dashboard/compute.py +++ b/dashboard/compute.py @@ -307,6 +307,20 @@ def indexed(name, rows, key): communities[community]["pooled_accuracy"], ) + levels = indexed("resource levels", d["resource_levels"], "group") + check("resource levels match groups", set(levels), {x["group"] for x in d["resource"]}) + for community, row in communities.items(): + pages = row.get("common_crawl_pages") + if pages is None: + continue + level = levels[row["language_resourcedness"]] + low, high = level["min_pages"], level["max_pages"] + check( + f"{community} Common Crawl pages within its level", + (low is None or pages >= low) and (high is None or pages < high), + True, + ) + matrix = d["riddle_matrix"] solved_by = matrix["solved_by"] check("riddle matrix models", matrix["models"], [row["model"] for row in d["leaderboard"]]) diff --git a/dashboard/template.html b/dashboard/template.html index bab99b4..7213eab 100644 --- a/dashboard/template.html +++ b/dashboard/template.html @@ -322,7 +322,7 @@

Cultural fingerprint

- +
@@ -361,7 +361,7 @@

How many models solve each riddle

Language resources

-

Each dot is a community, placed by the accuracy of all its scored answers pooled over models. Diamonds pool every answer in the level, with 95% intervals from resampling communities. Levels follow Common Crawl page counts for the language.

+

Each dot is a community, placed by the accuracy of all its scored answers pooled over models. Diamonds pool every answer in the level, with 95% intervals from resampling communities.

Source: Section 5 and Figure 8 of the paper.
@@ -528,6 +528,7 @@

All model configurations

region: c.un_m49_subregion, resource: c.language_resourcedness, writers: c.riddle_writers, mean: c.mean_accuracy, median_model: c.median_model, frontier: c.frontier_accuracy, small: c.small_accuracy, below10: c.n_below_10, above20: c.n_above_20, pooled: c.pooled_accuracy, rank_from_hardest: i + 1, + pages: c.common_crawl_pages, pages_note: c.common_crawl_note, })); const ci = Object.fromEntries(comms.map((c, i) => [c.id, i])); const cell = Object.fromEntries(R.cells.map(x => [x.model_display + '|' + x.community, x])); @@ -555,6 +556,7 @@

All model configurations

pairs: R.pairs, pair_models: R.pair_models, search_losses: R.search_losses, english: R.english, english_models: R.english_models, english_high: R.english_high_resource_accuracy, chinese: R.chinese, tier_effects: R.tier_effects, local_term: R.local_term_effect, coverage: R.coverage, frontier_models: R.frontier_models, small_models: R.small_models, links: R.links || {}, + levels: R.resource_levels, cc_archive: (R.common_crawl_archive || []).join(', '), }; } const $ = s => document.querySelector(s); @@ -571,6 +573,16 @@

All model configurations

const label = id => (byId[id] || {}).label || id; const MOBILE = () => window.innerWidth < 640; const MD = D.md; +const pagesFmt = n => n >= 1e6 ? (n / 1e6).toLocaleString('en-US', {maximumFractionDigits: 1}) + 'M' : n >= 1e3 ? Math.round(n / 1e3).toLocaleString('en-US') + 'K' : n0(n); +function levelRange(l) { + if (l.min_pages == null) return `under ${pagesFmt(l.max_pages)} pages, or not detected`; + if (l.max_pages == null) return `${pagesFmt(l.min_pages)} pages or more`; + return `${pagesFmt(l.min_pages)} to ${pagesFmt(l.max_pages)} pages`; +} +const RES_DEF = `Resource levels count the web pages in the language in Common Crawl${D.cc_archive ? ' (' + D.cc_archive + ')' : ''}: ` + + D.levels.map(l => `${RES_LABEL[l.group].toLowerCase()}, ${levelRange(l)}`).join('; ') + + `. Dialects take their main language's count; for example, every Arabic variety uses the count for Arabic.`; +document.getElementById('hm-res-btn').title = RES_DEF; document.querySelectorAll('[data-k="nhe"]').forEach(e => e.textContent = C.length); document.querySelectorAll('[data-k="nm"]').forEach(e => e.textContent = M.length); // Communities where no model reaches a quarter of the riddles. @@ -865,7 +877,8 @@

All model configurations

const c = byId[cur], j = c.i; const easyRank = C.length - c.rank_from_hardest + 1; $('#ex-name').textContent = c.label; - $('#ex-meta').innerHTML = `${c.language} · ${c.country} · ${c.region}${RES_LABEL[c.resource]} resource`; + const pagesText = c.pages == null ? '' : c.pages > 0 ? ` · about ${pagesFmt(c.pages)} Common Crawl pages` : ' · not detected in Common Crawl'; + $('#ex-meta').innerHTML = `${c.language} · ${c.country} · ${c.region}${RES_LABEL[c.resource]} resource${pagesText}`; $('#ex-link').href = '#community=' + cur; const vals = M.map((m, i) => D.grid[i][j]); const bi = vals.indexOf(Math.max(...vals)); @@ -908,6 +921,7 @@

All model configurations

/* ---------- resource ---------- */ register(function resource() { + $('#res-def').textContent = RES_DEF; const tiers = ['high', 'medium', 'low', 'extremely-low']; const R = Object.fromEntries(D.resource.map(r => [r.group, r])); const tr = []; @@ -919,7 +933,7 @@

All model configurations

hovertemplate: '%{text}
%{x:.1f}%'}); tr.push({type: 'scatter', mode: 'markers', x: [R[t].pooled], y: [k], error_x: {type: 'data', symmetric: false, array: [R[t].chi - R[t].pooled], arrayminus: [R[t].pooled - R[t].clo], color: css('--ink'), thickness: 2, width: 6}, marker: {symbol: 'diamond', size: 18, color: css('--ink'), line: {width: 2, color: css('--card')}}, - hovertemplate: `${RES_LABEL[t]} resource
pooled %{x:.1f}%
${R[t].communities} communities`}); + hovertemplate: `${RES_LABEL[t]} resource (${levelRange(D.levels.find(l => l.group === t))})
pooled %{x:.1f}%
${R[t].communities} communities`}); }); plot('resource', tr, base({height: 380, margin: {l: 10, r: 20, t: 6, b: 44}, xaxis: {range: [0, 80], ticksuffix: '%', gridcolor: css('--grid'), fixedrange: true, title: {text: 'accuracy', font: {size: 12}}}, diff --git a/data/derived/dashboard/dashboard_data.json b/data/derived/dashboard/dashboard_data.json index 3111495..b8cb17c 100644 --- a/data/derived/dashboard/dashboard_data.json +++ b/data/derived/dashboard/dashboard_data.json @@ -1683,7 +1683,9 @@ "country_code": "IDN", "language_resourcedness": "extremely-low", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 0.0, + "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)." }, { "community": "bugisnese_indonesia", @@ -1700,7 +1702,9 @@ "country_code": "IDN", "language_resourcedness": "extremely-low", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 0.0, + "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)." }, { "community": "lumasaba_uganda", @@ -1717,7 +1721,9 @@ "country_code": "UGA", "language_resourcedness": "extremely-low", "un_m49_subregion": "Sub-Saharan Africa", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 0.0, + "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)." }, { "community": "arabic_sudan", @@ -1734,7 +1740,9 @@ "country_code": "SDN", "language_resourcedness": "high", "un_m49_subregion": "North Africa", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 14286482.0, + "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2." }, { "community": "yoruba_nigeria", @@ -1751,7 +1759,9 @@ "country_code": "NGA", "language_resourcedness": "low", "un_m49_subregion": "Sub-Saharan Africa", - "riddle_writers": 4 + "riddle_writers": 4, + "common_crawl_pages": 36052.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "luganda_uganda", @@ -1768,7 +1778,9 @@ "country_code": "UGA", "language_resourcedness": "extremely-low", "un_m49_subregion": "Sub-Saharan Africa", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 7704.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "afaan_oromo_oromo", @@ -1785,7 +1797,9 @@ "country_code": "ETH", "language_resourcedness": "low", "un_m49_subregion": "Sub-Saharan Africa", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": null, + "common_crawl_note": null }, { "community": "swahili_kenya", @@ -1802,7 +1816,9 @@ "country_code": "KEN", "language_resourcedness": "medium", "un_m49_subregion": "Sub-Saharan Africa", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 249945.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "assamese_india", @@ -1819,7 +1835,9 @@ "country_code": "IND", "language_resourcedness": "low", "un_m49_subregion": "South Asia", - "riddle_writers": 4 + "riddle_writers": 4, + "common_crawl_pages": 66726.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "belarusian_belarus", @@ -1836,7 +1854,9 @@ "country_code": "BLR", "language_resourcedness": "medium", "un_m49_subregion": "Eastern Europe", - "riddle_writers": 4 + "riddle_writers": 4, + "common_crawl_pages": 367299.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "kannada_india", @@ -1853,7 +1873,9 @@ "country_code": "IND", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 323079.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "malayalam_india", @@ -1870,7 +1892,9 @@ "country_code": "IND", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 509416.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "konkani_india", @@ -1887,7 +1911,9 @@ "country_code": "IND", "language_resourcedness": "extremely-low", "un_m49_subregion": "South Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 0.0, + "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)." }, { "community": "hungarian_hungary", @@ -1904,7 +1930,9 @@ "country_code": "HUN", "language_resourcedness": "high", "un_m49_subregion": "Eastern Europe", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 11427802.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "chinese_malaysia", @@ -1921,7 +1949,9 @@ "country_code": "MYS", "language_resourcedness": "high", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 93776922.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "uzbek_uzbekistan", @@ -1938,7 +1968,9 @@ "country_code": "UZB", "language_resourcedness": "medium", "un_m49_subregion": "Central Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 669583.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "german_germany", @@ -1955,7 +1987,9 @@ "country_code": "DEU", "language_resourcedness": "high", "un_m49_subregion": "Western Europe", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 126414617.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "arabic_syria", @@ -1972,7 +2006,9 @@ "country_code": "SYR", "language_resourcedness": "high", "un_m49_subregion": "West Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 14286482.0, + "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2." }, { "community": "persian_iran", @@ -1989,7 +2025,9 @@ "country_code": "IRN", "language_resourcedness": "high", "un_m49_subregion": "South Asia", - "riddle_writers": 10 + "riddle_writers": 10, + "common_crawl_pages": 16267415.0, + "common_crawl_note": "Mapped from Iranian Persian (pes) to Persian macro-language (fas)." }, { "community": "thai_thailand", @@ -2006,7 +2044,9 @@ "country_code": "THA", "language_resourcedness": "medium", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 5 + "riddle_writers": 5, + "common_crawl_pages": 8002604.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "chinese_china", @@ -2023,7 +2063,9 @@ "country_code": "CHN", "language_resourcedness": "high", "un_m49_subregion": "East Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 93776922.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "spanish_argentina", @@ -2040,7 +2082,9 @@ "country_code": "ARG", "language_resourcedness": "high", "un_m49_subregion": "Latin America", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 98972317.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "cantonese_china", @@ -2057,7 +2101,9 @@ "country_code": "CHN", "language_resourcedness": "extremely-low", "un_m49_subregion": "East Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 0.0, + "common_crawl_note": "Not reported as a CLD2 primary language in this crawl; page count set to 0 (not proof of zero web pages)." }, { "community": "telugu_india", @@ -2074,7 +2120,9 @@ "country_code": "IND", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 8 + "riddle_writers": 8, + "common_crawl_pages": 425530.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "catalan_catalonia", @@ -2091,7 +2139,9 @@ "country_code": "ESP", "language_resourcedness": "medium", "un_m49_subregion": "Southern Europe", - "riddle_writers": 5 + "riddle_writers": 5, + "common_crawl_pages": 4383390.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "nepali_nepal", @@ -2108,7 +2158,9 @@ "country_code": "NPL", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 1274697.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "chinese_simplified_china", @@ -2125,7 +2177,9 @@ "country_code": "CHN", "language_resourcedness": "high", "un_m49_subregion": "East Asia", - "riddle_writers": 7 + "riddle_writers": 7, + "common_crawl_pages": 93776922.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "luxembourgish_luxembourg", @@ -2142,7 +2196,9 @@ "country_code": "LUX", "language_resourcedness": "low", "un_m49_subregion": "Western Europe", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 80957.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "russian_russia", @@ -2159,7 +2215,9 @@ "country_code": "RUS", "language_resourcedness": "high", "un_m49_subregion": "Eastern Europe", - "riddle_writers": 5 + "riddle_writers": 5, + "common_crawl_pages": 146902975.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "indonesian_indonesia", @@ -2176,7 +2234,9 @@ "country_code": "IDN", "language_resourcedness": "high", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 23924783.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "ukrainian_ukraine", @@ -2193,7 +2253,9 @@ "country_code": "UKR", "language_resourcedness": "high", "un_m49_subregion": "Eastern Europe", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 17911708.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "serbian_serbia", @@ -2210,7 +2272,9 @@ "country_code": "SRB", "language_resourcedness": "medium", "un_m49_subregion": "Southern Europe", - "riddle_writers": 4 + "riddle_writers": 4, + "common_crawl_pages": 5713950.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "polish_poland", @@ -2227,7 +2291,9 @@ "country_code": "POL", "language_resourcedness": "high", "un_m49_subregion": "Eastern Europe", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 44157233.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "urdu_pakistan", @@ -2244,7 +2310,9 @@ "country_code": "PAK", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 689187.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "arabic_morocco", @@ -2261,7 +2329,9 @@ "country_code": "MAR", "language_resourcedness": "high", "un_m49_subregion": "North Africa", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 14286482.0, + "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2." }, { "community": "tamil_india", @@ -2278,7 +2348,9 @@ "country_code": "IND", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 960601.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "marathi_india", @@ -2295,7 +2367,9 @@ "country_code": "IND", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 571814.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "arabic_egypt", @@ -2312,7 +2386,9 @@ "country_code": "EGY", "language_resourcedness": "high", "un_m49_subregion": "North Africa", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 14286482.0, + "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2." }, { "community": "bengali_bangladesh", @@ -2329,7 +2405,9 @@ "country_code": "BGD", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 2318326.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "bulgarian_bulgaria", @@ -2346,7 +2424,9 @@ "country_code": "BGR", "language_resourcedness": "medium", "un_m49_subregion": "Eastern Europe", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 6211572.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "italian_italy", @@ -2363,7 +2443,9 @@ "country_code": "ITA", "language_resourcedness": "high", "un_m49_subregion": "Southern Europe", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 52686575.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "tunisian_arabic_tunisia", @@ -2380,7 +2462,9 @@ "country_code": "TUN", "language_resourcedness": "high", "un_m49_subregion": "North Africa", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 14286482.0, + "common_crawl_note": "Mapped to Arabic macro-language; dialect is not separated by CLD2." }, { "community": "portuguese_portugal", @@ -2397,7 +2481,9 @@ "country_code": "PRT", "language_resourcedness": "high", "un_m49_subregion": "Southern Europe", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 54053402.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "vietnamese_vietnam", @@ -2414,7 +2500,9 @@ "country_code": "VNM", "language_resourcedness": "high", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 23000895.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "spanish_peru", @@ -2431,7 +2519,9 @@ "country_code": "PER", "language_resourcedness": "high", "un_m49_subregion": "Latin America", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 98972317.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "sinhala_sri_lanka", @@ -2448,7 +2538,9 @@ "country_code": "LKA", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 189589.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "bengali_india", @@ -2465,7 +2557,9 @@ "country_code": "IND", "language_resourcedness": "medium", "un_m49_subregion": "South Asia", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 2318326.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "spanish_cuba", @@ -2482,7 +2576,9 @@ "country_code": "CUB", "language_resourcedness": "high", "un_m49_subregion": "Caribbean", - "riddle_writers": 3 + "riddle_writers": 3, + "common_crawl_pages": 98972317.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "chinese_singapore", @@ -2499,7 +2595,9 @@ "country_code": "SGP", "language_resourcedness": "high", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 93776922.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "french_canada", @@ -2516,7 +2614,9 @@ "country_code": "CAN", "language_resourcedness": "high", "un_m49_subregion": "North America", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 103720803.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "malay_malaysia", @@ -2533,7 +2633,9 @@ "country_code": "MYS", "language_resourcedness": "medium", "un_m49_subregion": "Southeast Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 1840421.0, + "common_crawl_note": "Mapped from Standard Malay (zsm) to Malay macro-language (msa)." }, { "community": "english_canada", @@ -2550,7 +2652,9 @@ "country_code": "CAN", "language_resourcedness": "high", "un_m49_subregion": "North America", - "riddle_writers": 4 + "riddle_writers": 4, + "common_crawl_pages": 865530140.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "dutch_netherlands", @@ -2567,7 +2671,9 @@ "country_code": "NLD", "language_resourcedness": "high", "un_m49_subregion": "Western Europe", - "riddle_writers": 4 + "riddle_writers": 4, + "common_crawl_pages": 38004267.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "brazilian_portuguese", @@ -2584,7 +2690,9 @@ "country_code": "BRA", "language_resourcedness": "high", "un_m49_subregion": "Latin America", - "riddle_writers": 9 + "riddle_writers": 9, + "common_crawl_pages": 54053402.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "french_france", @@ -2601,7 +2709,9 @@ "country_code": "FRA", "language_resourcedness": "high", "un_m49_subregion": "Western Europe", - "riddle_writers": 2 + "riddle_writers": 2, + "common_crawl_pages": 103720803.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "japanese_japan", @@ -2618,7 +2728,9 @@ "country_code": "JPN", "language_resourcedness": "high", "un_m49_subregion": "East Asia", - "riddle_writers": 1 + "riddle_writers": 1, + "common_crawl_pages": 112295117.0, + "common_crawl_note": "Exact ISO 639-3 code match." }, { "community": "english_usa", @@ -2635,9 +2747,36 @@ "country_code": "USA", "language_resourcedness": "high", "un_m49_subregion": "North America", - "riddle_writers": 5 + "riddle_writers": 5, + "common_crawl_pages": 865530140.0, + "common_crawl_note": "Exact ISO 639-3 code match." } ], + "resource_levels": [ + { + "group": "high", + "min_pages": 10000000, + "max_pages": null + }, + { + "group": "medium", + "min_pages": 100000, + "max_pages": 10000000 + }, + { + "group": "low", + "min_pages": 10000, + "max_pages": 100000 + }, + { + "group": "extremely-low", + "min_pages": null, + "max_pages": 10000 + } + ], + "common_crawl_archive": [ + "CC-MAIN-2026-34" + ], "cells": [ { "model_display": "Amazon Nova Lite",