diff --git a/docs/TASKS.md b/docs/TASKS.md
index cfb5ae2c..e4dcd875 100644
--- a/docs/TASKS.md
+++ b/docs/TASKS.md
@@ -83,7 +83,7 @@ Languages are **derived in code** — there is no `languages` field to set in th
 YAML. A task resolves to a canonical [`lang_Script`](https://en.wikipedia.org/wiki/IETF_language_tag)
 code (e.g. `deu_Latn`) from, in order:
 
-1. **`flores200:src-tgt` task names** → the non-English side(s) of the pair.
+1. **`flores200:src-tgt` (and `flores200_src-tgt_bpb`) task names** → the non-English side(s) of the pair.
 2. **The `{lang}` value** substituted into a `valid_langs` template (preferred
    for new multilingual groups — see the template expansion above).
 3. **The task's `subset`** (e.g. `de`, `german`, `deu_Latn` all fold to
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_belebele_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_belebele_bpb_template_yaml
new file mode 100644
index 00000000..f8494d0a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_belebele_bpb_template_yaml
@@ -0,0 +1,17 @@
+# Belebele in cloze form: the continuation is the answer text rather than the
+# letter "A".."D", so BPB measures comprehension instead of format.
+# Per-language tasks live in belebele/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - belebele_bpb
+dataset_path: facebook/belebele
+test_split: test
+fewshot_split: test
+fewshot_config:
+  sampler: first_n
+doc_to_text: "{{flores_passage}}\nQuestion: {{question.strip()}}\nAnswer:"
+doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}"
+doc_to_choice: "{{[mc_answer1, mc_answer2, mc_answer3, mc_answer4]}}"
+process_results: !function utils.process_results_belebele
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_bpb_template_yaml
new file mode 100644
index 00000000..81c14789
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_bpb_template_yaml
@@ -0,0 +1,22 @@
+# Shared config for the BPB (bits-per-byte) task variants.
+#
+# Each task keeps its upstream prompt format and dataset so that BPB is
+# measured on exactly the same documents as the accuracy-based task, but
+# overrides process_results (see utils.py) so that the gold continuation's
+# (loglikelihood, byte_length) pair is emitted for the bits_per_byte
+# aggregation. acc/acc_norm come along for free.
+tag:
+  - bpb
+output_type: multiple_choice
+metric_list:
+  - metric: bits_per_byte
+    aggregation: bits_per_byte
+    higher_is_better: false
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_bpb_template_yaml
new file mode 100644
index 00000000..819f6b23
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_bpb_template_yaml
@@ -0,0 +1,21 @@
+# FLORES-200: BPB of the reference translation given the source sentence,
+# using the same "DE: <source> EN:" prompt shape as lighteval's flores200 CF
+# formulation. Included by the two direction templates below; per-pair tasks
+# live in flores200/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - flores200_bpb
+dataset_path: facebook/flores
+test_split: devtest
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+output_type: loglikelihood
+# doc_to_target supplies the leading space itself (see utils.py).
+target_delimiter: ""
+metric_list:
+  - metric: bits_per_byte
+    aggregation: bits_per_byte
+    higher_is_better: false
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_eng_xx_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_eng_xx_bpb_template_yaml
new file mode 100644
index 00000000..5845c54d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_eng_xx_bpb_template_yaml
@@ -0,0 +1,3 @@
+include: _flores200_bpb_template_yaml
+doc_to_target: !function utils.doc_to_target_flores_eng_xx
+process_results: !function utils.process_results_flores_eng_xx
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_xx_eng_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_xx_eng_bpb_template_yaml
new file mode 100644
index 00000000..a7bea989
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_flores200_xx_eng_bpb_template_yaml
@@ -0,0 +1,3 @@
+include: _flores200_bpb_template_yaml
+doc_to_target: !function utils.doc_to_target_flores_xx_eng
+process_results: !function utils.process_results_flores_xx_eng
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_global_mmlu_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_global_mmlu_bpb_template_yaml
new file mode 100644
index 00000000..5fc3ef3f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_global_mmlu_bpb_template_yaml
@@ -0,0 +1,18 @@
+# Global-MMLU in cloze form, like _mmlu_stem_bpb_template_yaml. Each language
+# is a single task over the whole test split (all 57 subjects), so BPB is one
+# byte-weighted corpus aggregate per language. Per-language tasks live in
+# global_mmlu/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - global_mmlu_bpb
+dataset_path: CohereForAI/Global-MMLU
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+doc_to_text: "Question: {{question.strip()}}\nAnswer:"
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer.strip())}}"
+doc_to_choice: "{{[option_a, option_b, option_c, option_d]}}"
+process_results: !function utils.process_results_global_mmlu
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_include_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_include_bpb_template_yaml
new file mode 100644
index 00000000..43d70ec5
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_include_bpb_template_yaml
@@ -0,0 +1,14 @@
+# INCLUDE in cloze form, like _mmlu_stem_bpb_template_yaml. Each language is a
+# single task over the whole test split (all domains). Per-language tasks live
+# in include/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - include_bpb
+dataset_path: CohereForAI/include-base-44
+test_split: test
+doc_to_text: "Question: {{question.strip()}}\nAnswer:"
+doc_to_target: answer
+doc_to_choice: "{{[option_a, option_b, option_c, option_d]}}"
+process_results: !function utils.process_results_include
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_mgsm_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_mgsm_bpb_template_yaml
new file mode 100644
index 00000000..781f2533
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_mgsm_bpb_template_yaml
@@ -0,0 +1,20 @@
+# MGSM: BPB of the final answer number after the upstream lm-eval
+# mgsm_direct_<lang> prompt (the test split has no worked solutions).
+# Per-language tasks live in mgsm/ and carry the localized prompt.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - mgsm_bpb
+dataset_path: jbross-ibm-research/mgsm
+training_split: train
+test_split: test
+output_type: loglikelihood
+# doc_to_target supplies the leading space itself (see utils.py).
+target_delimiter: ""
+doc_to_target: !function utils.doc_to_target_mgsm
+process_results: !function utils.process_results_mgsm
+metric_list:
+  - metric: bits_per_byte
+    aggregation: bits_per_byte
+    higher_is_better: false
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_mmlu_stem_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_mmlu_stem_bpb_template_yaml
new file mode 100644
index 00000000..e6d70fa4
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_mmlu_stem_bpb_template_yaml
@@ -0,0 +1,18 @@
+# MMLU in cloze/continuation form: the continuation is the answer text rather
+# than the letter "A".."D", so BPB measures knowledge instead of format.
+# Every subject carries the `mmlu_stem_bpb` tag, so a single
+# `--tasks mmlu_stem_bpb` invocation runs all 19 STEM subjects in one process
+# and writes them to one results JSON.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - mmlu_stem_bpb
+dataset_path: cais/mmlu
+test_split: test
+fewshot_split: dev
+fewshot_config:
+  sampler: first_n
+doc_to_text: "Question: {{question.strip()}}\nAnswer:"
+doc_to_target: "{{answer}}"
+doc_to_choice: "{{choices}}"
+process_results: !function utils.process_results_mmlu
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_xcopa_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_xcopa_bpb_template_yaml
new file mode 100644
index 00000000..d689d5e9
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_xcopa_bpb_template_yaml
@@ -0,0 +1,13 @@
+# XCOPA with the upstream lm-eval prompt; the per-language connector comes from
+# each task's doc_to_text. Per-language tasks live in xcopa/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - xcopa_bpb
+dataset_path: cambridgeltl/xcopa
+validation_split: validation
+test_split: test
+doc_to_target: label
+doc_to_choice: !function utils.doc_to_choice_xcopa
+process_results: !function utils.process_results_xcopa
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_xstorycloze_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_xstorycloze_bpb_template_yaml
new file mode 100644
index 00000000..4fe4a804
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_xstorycloze_bpb_template_yaml
@@ -0,0 +1,14 @@
+# XStoryCloze with the upstream lm-eval prompt. Per-language tasks live in
+# xstorycloze/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - xstorycloze_bpb
+dataset_path: juletxara/xstory_cloze
+training_split: train
+validation_split: eval
+doc_to_text: "{{[input_sentence_1, input_sentence_2, input_sentence_3, input_sentence_4]|join(' ')}}"
+doc_to_target: "{{answer_right_ending-1}}"
+doc_to_choice: "{{[sentence_quiz1, sentence_quiz2]}}"
+process_results: !function utils.process_results_xstorycloze
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/_xwinograd_bpb_template_yaml b/oellm/resources/custom_lm_eval_tasks/bpb/_xwinograd_bpb_template_yaml
new file mode 100644
index 00000000..e9e3f145
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/_xwinograd_bpb_template_yaml
@@ -0,0 +1,14 @@
+# XWinograd shares the Winogrande schema, so it reuses winogrande_bpb's
+# multiple_input functions: BPB is measured on the shared continuation.
+# Per-language tasks live in xwinograd/.
+include: _bpb_template_yaml
+tag:
+  - bpb
+  - bpb_multilingual
+  - xwinograd_bpb
+dataset_path: Muennighoff/xwinograd
+test_split: test
+doc_to_text: !function utils.doc_to_text_winogrande
+doc_to_target: !function utils.doc_to_target_winogrande
+doc_to_choice: !function utils.doc_to_choice_winogrande
+process_results: !function utils.process_results_winogrande
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/arc_challenge_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/arc_challenge_bpb.yaml
new file mode 100644
index 00000000..a2a80c99
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/arc_challenge_bpb.yaml
@@ -0,0 +1,11 @@
+include: _bpb_template_yaml
+task: arc_challenge_bpb
+dataset_path: allenai/ai2_arc
+dataset_name: ARC-Challenge
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+process_results: !function utils.process_results_arc
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/arc_easy_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/arc_easy_bpb.yaml
new file mode 100644
index 00000000..d4e8f437
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/arc_easy_bpb.yaml
@@ -0,0 +1,11 @@
+include: _bpb_template_yaml
+task: arc_easy_bpb
+dataset_path: allenai/ai2_arc
+dataset_name: ARC-Easy
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+process_results: !function utils.process_results_arc
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_bul_Cyrl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_bul_Cyrl_bpb.yaml
new file mode 100644
index 00000000..bde960f5
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_bul_Cyrl_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_bul_Cyrl_bpb
+dataset_name: bul_Cyrl
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ces_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ces_Latn_bpb.yaml
new file mode 100644
index 00000000..91f3f984
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ces_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_ces_Latn_bpb
+dataset_name: ces_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_dan_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_dan_Latn_bpb.yaml
new file mode 100644
index 00000000..2d09e0cf
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_dan_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_dan_Latn_bpb
+dataset_name: dan_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_deu_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_deu_Latn_bpb.yaml
new file mode 100644
index 00000000..6be6e913
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_deu_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_deu_Latn_bpb
+dataset_name: deu_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ell_Grek_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ell_Grek_bpb.yaml
new file mode 100644
index 00000000..6467ed0b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ell_Grek_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_ell_Grek_bpb
+dataset_name: ell_Grek
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_eng_Latn_bpb.yaml
new file mode 100644
index 00000000..5e418e6b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_eng_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_eng_Latn_bpb
+dataset_name: eng_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_est_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_est_Latn_bpb.yaml
new file mode 100644
index 00000000..fa3e7798
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_est_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_est_Latn_bpb
+dataset_name: est_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_fin_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_fin_Latn_bpb.yaml
new file mode 100644
index 00000000..e0205942
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_fin_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_fin_Latn_bpb
+dataset_name: fin_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_fra_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_fra_Latn_bpb.yaml
new file mode 100644
index 00000000..2932d13c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_fra_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_fra_Latn_bpb
+dataset_name: fra_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_hrv_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_hrv_Latn_bpb.yaml
new file mode 100644
index 00000000..988c8ba3
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_hrv_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_hrv_Latn_bpb
+dataset_name: hrv_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_hun_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_hun_Latn_bpb.yaml
new file mode 100644
index 00000000..c94cd8ff
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_hun_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_hun_Latn_bpb
+dataset_name: hun_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ita_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ita_Latn_bpb.yaml
new file mode 100644
index 00000000..981a531f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ita_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_ita_Latn_bpb
+dataset_name: ita_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_lit_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_lit_Latn_bpb.yaml
new file mode 100644
index 00000000..c9208cb9
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_lit_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_lit_Latn_bpb
+dataset_name: lit_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_lvs_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_lvs_Latn_bpb.yaml
new file mode 100644
index 00000000..e8d1e17d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_lvs_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_lvs_Latn_bpb
+dataset_name: lvs_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_mlt_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_mlt_Latn_bpb.yaml
new file mode 100644
index 00000000..dbaec74d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_mlt_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_mlt_Latn_bpb
+dataset_name: mlt_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_nld_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_nld_Latn_bpb.yaml
new file mode 100644
index 00000000..2a9227d2
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_nld_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_nld_Latn_bpb
+dataset_name: nld_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_nob_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_nob_Latn_bpb.yaml
new file mode 100644
index 00000000..af49436d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_nob_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_nob_Latn_bpb
+dataset_name: nob_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_pol_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_pol_Latn_bpb.yaml
new file mode 100644
index 00000000..d05cd88e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_pol_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_pol_Latn_bpb
+dataset_name: pol_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_por_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_por_Latn_bpb.yaml
new file mode 100644
index 00000000..5cc81e60
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_por_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_por_Latn_bpb
+dataset_name: por_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ron_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ron_Latn_bpb.yaml
new file mode 100644
index 00000000..57ef7b79
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_ron_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_ron_Latn_bpb
+dataset_name: ron_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_slk_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_slk_Latn_bpb.yaml
new file mode 100644
index 00000000..538513fc
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_slk_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_slk_Latn_bpb
+dataset_name: slk_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_slv_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_slv_Latn_bpb.yaml
new file mode 100644
index 00000000..ef601d41
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_slv_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_slv_Latn_bpb
+dataset_name: slv_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_spa_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_spa_Latn_bpb.yaml
new file mode 100644
index 00000000..4e145035
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_spa_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_spa_Latn_bpb
+dataset_name: spa_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_swe_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_swe_Latn_bpb.yaml
new file mode 100644
index 00000000..368037f2
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/belebele/belebele_swe_Latn_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_belebele_bpb_template_yaml
+task: belebele_swe_Latn_bpb
+dataset_name: swe_Latn
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/boolq_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/boolq_bpb.yaml
new file mode 100644
index 00000000..7871226d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/boolq_bpb.yaml
@@ -0,0 +1,10 @@
+include: _bpb_template_yaml
+task: boolq_bpb
+dataset_path: aps/super_glue
+dataset_name: boolq
+training_split: train
+validation_split: validation
+doc_to_text: "{{passage}}\nQuestion: {{question}}?\nAnswer:"
+doc_to_target: label
+doc_to_choice: ["no", "yes"]
+process_results: !function utils.process_results_boolq
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/commonsense_qa_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/commonsense_qa_bpb.yaml
new file mode 100644
index 00000000..60ba6b65
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/commonsense_qa_bpb.yaml
@@ -0,0 +1,11 @@
+# Cloze form: the continuation is the answer *text*, not the letter "A".."E".
+# Scoring BPB on a single letter would measure format-following, not knowledge.
+include: _bpb_template_yaml
+task: commonsense_qa_bpb
+dataset_path: tau/commonsense_qa
+training_split: train
+validation_split: validation
+doc_to_text: "Question: {{ question.strip() }}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+process_results: !function utils.process_results_commonsense_qa
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/copa_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/copa_bpb.yaml
new file mode 100644
index 00000000..4f21fe1c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/copa_bpb.yaml
@@ -0,0 +1,10 @@
+include: _bpb_template_yaml
+task: copa_bpb
+dataset_path: aps/super_glue
+dataset_name: copa
+training_split: train
+validation_split: validation
+doc_to_text: !function utils.doc_to_text_copa
+doc_to_target: !function utils.doc_to_target_copa
+doc_to_choice: !function utils.doc_to_choice_copa
+process_results: !function utils.process_results_copa
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_als_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_als_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..29c1c19c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_als_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_als_Latn-eng_Latn_bpb
+dataset_name: als_Latn-eng_Latn
+doc_to_text: 'SQ: {{sentence_als_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_bos_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_bos_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..30cbebe5
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_bos_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_bos_Latn-eng_Latn_bpb
+dataset_name: bos_Latn-eng_Latn
+doc_to_text: 'BS: {{sentence_bos_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_bul_Cyrl-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_bul_Cyrl-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..0d6c9846
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_bul_Cyrl-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_bul_Cyrl-eng_Latn_bpb
+dataset_name: bul_Cyrl-eng_Latn
+doc_to_text: 'BG: {{sentence_bul_Cyrl}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_cat_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_cat_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..49d8781e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_cat_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_cat_Latn-eng_Latn_bpb
+dataset_name: cat_Latn-eng_Latn
+doc_to_text: 'CA: {{sentence_cat_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ces_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ces_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..92d87031
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ces_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_ces_Latn-eng_Latn_bpb
+dataset_name: ces_Latn-eng_Latn
+doc_to_text: 'CS: {{sentence_ces_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_dan_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_dan_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..4bb1f91e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_dan_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_dan_Latn-eng_Latn_bpb
+dataset_name: dan_Latn-eng_Latn
+doc_to_text: 'DA: {{sentence_dan_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_deu_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_deu_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..96c398bd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_deu_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_deu_Latn-eng_Latn_bpb
+dataset_name: deu_Latn-eng_Latn
+doc_to_text: 'DE: {{sentence_deu_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ell_Grek-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ell_Grek-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..bac96b37
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ell_Grek-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_ell_Grek-eng_Latn_bpb
+dataset_name: ell_Grek-eng_Latn
+doc_to_text: 'EL: {{sentence_ell_Grek}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-als_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-als_Latn_bpb.yaml
new file mode 100644
index 00000000..7c08a96e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-als_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-als_Latn_bpb
+dataset_name: eng_Latn-als_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} SQ:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-bos_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-bos_Latn_bpb.yaml
new file mode 100644
index 00000000..32229e52
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-bos_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-bos_Latn_bpb
+dataset_name: eng_Latn-bos_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} BS:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-bul_Cyrl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-bul_Cyrl_bpb.yaml
new file mode 100644
index 00000000..cc45c194
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-bul_Cyrl_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-bul_Cyrl_bpb
+dataset_name: eng_Latn-bul_Cyrl
+doc_to_text: 'EN: {{sentence_eng_Latn}} BG:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-cat_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-cat_Latn_bpb.yaml
new file mode 100644
index 00000000..afe4ac31
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-cat_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-cat_Latn_bpb
+dataset_name: eng_Latn-cat_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} CA:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ces_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ces_Latn_bpb.yaml
new file mode 100644
index 00000000..afc09020
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ces_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-ces_Latn_bpb
+dataset_name: eng_Latn-ces_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} CS:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-dan_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-dan_Latn_bpb.yaml
new file mode 100644
index 00000000..962ccb5b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-dan_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-dan_Latn_bpb
+dataset_name: eng_Latn-dan_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} DA:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-deu_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-deu_Latn_bpb.yaml
new file mode 100644
index 00000000..ce10dcda
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-deu_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-deu_Latn_bpb
+dataset_name: eng_Latn-deu_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} DE:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ell_Grek_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ell_Grek_bpb.yaml
new file mode 100644
index 00000000..2e8c4906
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ell_Grek_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-ell_Grek_bpb
+dataset_name: eng_Latn-ell_Grek
+doc_to_text: 'EN: {{sentence_eng_Latn}} EL:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-est_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-est_Latn_bpb.yaml
new file mode 100644
index 00000000..b44c1be4
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-est_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-est_Latn_bpb
+dataset_name: eng_Latn-est_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} ET:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-eus_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-eus_Latn_bpb.yaml
new file mode 100644
index 00000000..7dd4b126
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-eus_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-eus_Latn_bpb
+dataset_name: eng_Latn-eus_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} EU:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-fin_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-fin_Latn_bpb.yaml
new file mode 100644
index 00000000..9b8e69c8
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-fin_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-fin_Latn_bpb
+dataset_name: eng_Latn-fin_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} FI:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-fra_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-fra_Latn_bpb.yaml
new file mode 100644
index 00000000..07f7ed90
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-fra_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-fra_Latn_bpb
+dataset_name: eng_Latn-fra_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} FR:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-gle_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-gle_Latn_bpb.yaml
new file mode 100644
index 00000000..10db051e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-gle_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-gle_Latn_bpb
+dataset_name: eng_Latn-gle_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} GA:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-glg_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-glg_Latn_bpb.yaml
new file mode 100644
index 00000000..d0709825
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-glg_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-glg_Latn_bpb
+dataset_name: eng_Latn-glg_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} GL:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-hrv_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-hrv_Latn_bpb.yaml
new file mode 100644
index 00000000..79df786a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-hrv_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-hrv_Latn_bpb
+dataset_name: eng_Latn-hrv_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} HR:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-hun_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-hun_Latn_bpb.yaml
new file mode 100644
index 00000000..6f69d714
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-hun_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-hun_Latn_bpb
+dataset_name: eng_Latn-hun_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} HU:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-isl_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-isl_Latn_bpb.yaml
new file mode 100644
index 00000000..892ca918
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-isl_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-isl_Latn_bpb
+dataset_name: eng_Latn-isl_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} IS:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ita_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ita_Latn_bpb.yaml
new file mode 100644
index 00000000..95cbbf0e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ita_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-ita_Latn_bpb
+dataset_name: eng_Latn-ita_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} IT:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-kat_Geor_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-kat_Geor_bpb.yaml
new file mode 100644
index 00000000..2a787cc0
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-kat_Geor_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-kat_Geor_bpb
+dataset_name: eng_Latn-kat_Geor
+doc_to_text: 'EN: {{sentence_eng_Latn}} KA:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-lit_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-lit_Latn_bpb.yaml
new file mode 100644
index 00000000..ae27ca7a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-lit_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-lit_Latn_bpb
+dataset_name: eng_Latn-lit_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} LT:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-lvs_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-lvs_Latn_bpb.yaml
new file mode 100644
index 00000000..c33180d3
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-lvs_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-lvs_Latn_bpb
+dataset_name: eng_Latn-lvs_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} LV:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-mkd_Cyrl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-mkd_Cyrl_bpb.yaml
new file mode 100644
index 00000000..07d59193
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-mkd_Cyrl_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-mkd_Cyrl_bpb
+dataset_name: eng_Latn-mkd_Cyrl
+doc_to_text: 'EN: {{sentence_eng_Latn}} MK:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-mlt_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-mlt_Latn_bpb.yaml
new file mode 100644
index 00000000..e22a1dc9
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-mlt_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-mlt_Latn_bpb
+dataset_name: eng_Latn-mlt_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} MT:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-nld_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-nld_Latn_bpb.yaml
new file mode 100644
index 00000000..765f534b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-nld_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-nld_Latn_bpb
+dataset_name: eng_Latn-nld_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} NL:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-nob_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-nob_Latn_bpb.yaml
new file mode 100644
index 00000000..d5ca5a8d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-nob_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-nob_Latn_bpb
+dataset_name: eng_Latn-nob_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} NB:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-pol_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-pol_Latn_bpb.yaml
new file mode 100644
index 00000000..85960d32
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-pol_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-pol_Latn_bpb
+dataset_name: eng_Latn-pol_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} PL:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-por_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-por_Latn_bpb.yaml
new file mode 100644
index 00000000..2dad20f0
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-por_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-por_Latn_bpb
+dataset_name: eng_Latn-por_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} PT:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ron_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ron_Latn_bpb.yaml
new file mode 100644
index 00000000..c1a62802
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ron_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-ron_Latn_bpb
+dataset_name: eng_Latn-ron_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} RO:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-slk_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-slk_Latn_bpb.yaml
new file mode 100644
index 00000000..6f9806af
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-slk_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-slk_Latn_bpb
+dataset_name: eng_Latn-slk_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} SK:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-slv_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-slv_Latn_bpb.yaml
new file mode 100644
index 00000000..dbc8f328
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-slv_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-slv_Latn_bpb
+dataset_name: eng_Latn-slv_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} SL:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-spa_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-spa_Latn_bpb.yaml
new file mode 100644
index 00000000..908aa144
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-spa_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-spa_Latn_bpb
+dataset_name: eng_Latn-spa_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} ES:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-srp_Cyrl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-srp_Cyrl_bpb.yaml
new file mode 100644
index 00000000..29a46df2
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-srp_Cyrl_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-srp_Cyrl_bpb
+dataset_name: eng_Latn-srp_Cyrl
+doc_to_text: 'EN: {{sentence_eng_Latn}} SR:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-swe_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-swe_Latn_bpb.yaml
new file mode 100644
index 00000000..83610761
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-swe_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-swe_Latn_bpb
+dataset_name: eng_Latn-swe_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} SV:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-tur_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-tur_Latn_bpb.yaml
new file mode 100644
index 00000000..9a5c80b6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-tur_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-tur_Latn_bpb
+dataset_name: eng_Latn-tur_Latn
+doc_to_text: 'EN: {{sentence_eng_Latn}} TR:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ukr_Cyrl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ukr_Cyrl_bpb.yaml
new file mode 100644
index 00000000..8e8e0a7f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eng_Latn-ukr_Cyrl_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_eng_xx_bpb_template_yaml
+task: flores200_eng_Latn-ukr_Cyrl_bpb
+dataset_name: eng_Latn-ukr_Cyrl
+doc_to_text: 'EN: {{sentence_eng_Latn}} UK:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_est_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_est_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..7b7fd709
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_est_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_est_Latn-eng_Latn_bpb
+dataset_name: est_Latn-eng_Latn
+doc_to_text: 'ET: {{sentence_est_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eus_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eus_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..40a70799
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_eus_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_eus_Latn-eng_Latn_bpb
+dataset_name: eus_Latn-eng_Latn
+doc_to_text: 'EU: {{sentence_eus_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_fin_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_fin_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..8d99b504
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_fin_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_fin_Latn-eng_Latn_bpb
+dataset_name: fin_Latn-eng_Latn
+doc_to_text: 'FI: {{sentence_fin_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_fra_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_fra_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..bfbd8ccd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_fra_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_fra_Latn-eng_Latn_bpb
+dataset_name: fra_Latn-eng_Latn
+doc_to_text: 'FR: {{sentence_fra_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_gle_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_gle_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..5f10723d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_gle_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_gle_Latn-eng_Latn_bpb
+dataset_name: gle_Latn-eng_Latn
+doc_to_text: 'GA: {{sentence_gle_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_glg_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_glg_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..37b5d9b3
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_glg_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_glg_Latn-eng_Latn_bpb
+dataset_name: glg_Latn-eng_Latn
+doc_to_text: 'GL: {{sentence_glg_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_hrv_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_hrv_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..f8123d2c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_hrv_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_hrv_Latn-eng_Latn_bpb
+dataset_name: hrv_Latn-eng_Latn
+doc_to_text: 'HR: {{sentence_hrv_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_hun_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_hun_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..fb4ad0cc
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_hun_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_hun_Latn-eng_Latn_bpb
+dataset_name: hun_Latn-eng_Latn
+doc_to_text: 'HU: {{sentence_hun_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_isl_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_isl_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..9bf5e2f7
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_isl_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_isl_Latn-eng_Latn_bpb
+dataset_name: isl_Latn-eng_Latn
+doc_to_text: 'IS: {{sentence_isl_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ita_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ita_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..4c092131
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ita_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_ita_Latn-eng_Latn_bpb
+dataset_name: ita_Latn-eng_Latn
+doc_to_text: 'IT: {{sentence_ita_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_kat_Geor-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_kat_Geor-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..638c1bc8
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_kat_Geor-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_kat_Geor-eng_Latn_bpb
+dataset_name: kat_Geor-eng_Latn
+doc_to_text: 'KA: {{sentence_kat_Geor}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_lit_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_lit_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..c86d9e5b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_lit_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_lit_Latn-eng_Latn_bpb
+dataset_name: lit_Latn-eng_Latn
+doc_to_text: 'LT: {{sentence_lit_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_lvs_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_lvs_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..7f8d87a4
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_lvs_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_lvs_Latn-eng_Latn_bpb
+dataset_name: lvs_Latn-eng_Latn
+doc_to_text: 'LV: {{sentence_lvs_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_mkd_Cyrl-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_mkd_Cyrl-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..265e4d7a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_mkd_Cyrl-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_mkd_Cyrl-eng_Latn_bpb
+dataset_name: mkd_Cyrl-eng_Latn
+doc_to_text: 'MK: {{sentence_mkd_Cyrl}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_mlt_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_mlt_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..ae4bbace
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_mlt_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_mlt_Latn-eng_Latn_bpb
+dataset_name: mlt_Latn-eng_Latn
+doc_to_text: 'MT: {{sentence_mlt_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_nld_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_nld_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..6031d85f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_nld_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_nld_Latn-eng_Latn_bpb
+dataset_name: nld_Latn-eng_Latn
+doc_to_text: 'NL: {{sentence_nld_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_nob_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_nob_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..698b2278
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_nob_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_nob_Latn-eng_Latn_bpb
+dataset_name: nob_Latn-eng_Latn
+doc_to_text: 'NB: {{sentence_nob_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_pol_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_pol_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..1a5f33bd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_pol_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_pol_Latn-eng_Latn_bpb
+dataset_name: pol_Latn-eng_Latn
+doc_to_text: 'PL: {{sentence_pol_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_por_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_por_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..fb73abc7
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_por_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_por_Latn-eng_Latn_bpb
+dataset_name: por_Latn-eng_Latn
+doc_to_text: 'PT: {{sentence_por_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ron_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ron_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..48abd04f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ron_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_ron_Latn-eng_Latn_bpb
+dataset_name: ron_Latn-eng_Latn
+doc_to_text: 'RO: {{sentence_ron_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_slk_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_slk_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..91af5e6c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_slk_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_slk_Latn-eng_Latn_bpb
+dataset_name: slk_Latn-eng_Latn
+doc_to_text: 'SK: {{sentence_slk_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_slv_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_slv_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..75bc0fd5
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_slv_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_slv_Latn-eng_Latn_bpb
+dataset_name: slv_Latn-eng_Latn
+doc_to_text: 'SL: {{sentence_slv_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_spa_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_spa_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..38cd3494
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_spa_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_spa_Latn-eng_Latn_bpb
+dataset_name: spa_Latn-eng_Latn
+doc_to_text: 'ES: {{sentence_spa_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_srp_Cyrl-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_srp_Cyrl-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..71577b93
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_srp_Cyrl-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_srp_Cyrl-eng_Latn_bpb
+dataset_name: srp_Cyrl-eng_Latn
+doc_to_text: 'SR: {{sentence_srp_Cyrl}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_swe_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_swe_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..9e791957
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_swe_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_swe_Latn-eng_Latn_bpb
+dataset_name: swe_Latn-eng_Latn
+doc_to_text: 'SV: {{sentence_swe_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_tur_Latn-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_tur_Latn-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..75f01837
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_tur_Latn-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_tur_Latn-eng_Latn_bpb
+dataset_name: tur_Latn-eng_Latn
+doc_to_text: 'TR: {{sentence_tur_Latn}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ukr_Cyrl-eng_Latn_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ukr_Cyrl-eng_Latn_bpb.yaml
new file mode 100644
index 00000000..8484fb44
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/flores200/flores200_ukr_Cyrl-eng_Latn_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_flores200_xx_eng_bpb_template_yaml
+task: flores200_ukr_Cyrl-eng_Latn_bpb
+dataset_name: ukr_Cyrl-eng_Latn
+doc_to_text: 'UK: {{sentence_ukr_Cyrl}} EN:'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_cs_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_cs_bpb.yaml
new file mode 100644
index 00000000..8a135368
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_cs_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_cs_bpb
+dataset_name: cs
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_de_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_de_bpb.yaml
new file mode 100644
index 00000000..4e3d2fdd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_de_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_de_bpb
+dataset_name: de
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_el_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_el_bpb.yaml
new file mode 100644
index 00000000..867466ef
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_el_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_el_bpb
+dataset_name: el
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_en_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_en_bpb.yaml
new file mode 100644
index 00000000..182d317c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_en_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_en_bpb
+dataset_name: en
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_es_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_es_bpb.yaml
new file mode 100644
index 00000000..c3eae381
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_es_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_es_bpb
+dataset_name: es
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_fr_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_fr_bpb.yaml
new file mode 100644
index 00000000..bb4f741a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_fr_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_fr_bpb
+dataset_name: fr
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_he_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_he_bpb.yaml
new file mode 100644
index 00000000..39eb4d3a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_he_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_he_bpb
+dataset_name: he
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_it_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_it_bpb.yaml
new file mode 100644
index 00000000..55ad795d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_it_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_it_bpb
+dataset_name: it
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_lt_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_lt_bpb.yaml
new file mode 100644
index 00000000..a350fdd2
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_lt_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_lt_bpb
+dataset_name: lt
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_nl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_nl_bpb.yaml
new file mode 100644
index 00000000..4b2d2f97
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_nl_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_nl_bpb
+dataset_name: nl
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_pl_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_pl_bpb.yaml
new file mode 100644
index 00000000..4cac8955
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_pl_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_pl_bpb
+dataset_name: pl
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_pt_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_pt_bpb.yaml
new file mode 100644
index 00000000..7f9accae
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_pt_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_pt_bpb
+dataset_name: pt
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_ro_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_ro_bpb.yaml
new file mode 100644
index 00000000..90fb1143
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_ro_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_ro_bpb
+dataset_name: ro
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_ru_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_ru_bpb.yaml
new file mode 100644
index 00000000..993bb565
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_ru_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_ru_bpb
+dataset_name: ru
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_sr_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_sr_bpb.yaml
new file mode 100644
index 00000000..8adcea91
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_sr_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_sr_bpb
+dataset_name: sr
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_sv_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_sv_bpb.yaml
new file mode 100644
index 00000000..eb1c1f89
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_sv_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_sv_bpb
+dataset_name: sv
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_tr_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_tr_bpb.yaml
new file mode 100644
index 00000000..fe0ce6d6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_tr_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_tr_bpb
+dataset_name: tr
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_uk_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_uk_bpb.yaml
new file mode 100644
index 00000000..ee29b9e9
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/global_mmlu/global_mmlu_uk_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_global_mmlu_bpb_template_yaml
+task: global_mmlu_uk_bpb
+dataset_name: uk
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/hellaswag_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/hellaswag_bpb.yaml
new file mode 100644
index 00000000..00cb7866
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/hellaswag_bpb.yaml
@@ -0,0 +1,12 @@
+include: _bpb_template_yaml
+task: hellaswag_bpb
+dataset_path: Rowan/hellaswag
+dataset_name: null
+training_split: train
+validation_split: validation
+test_split: null
+process_docs: !function utils.process_docs_hellaswag
+doc_to_text: "{{query}}"
+doc_to_target: "{{gold}}"
+doc_to_choice: "choices"
+process_results: !function utils.process_results_hellaswag
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_albanian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_albanian_bpb.yaml
new file mode 100644
index 00000000..457f76f8
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_albanian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_albanian_bpb
+dataset_name: Albanian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_armenian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_armenian_bpb.yaml
new file mode 100644
index 00000000..e60a051e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_armenian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_armenian_bpb
+dataset_name: Armenian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_azerbaijani_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_azerbaijani_bpb.yaml
new file mode 100644
index 00000000..f42aa978
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_azerbaijani_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_azerbaijani_bpb
+dataset_name: Azerbaijani
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_basque_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_basque_bpb.yaml
new file mode 100644
index 00000000..102ef8a6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_basque_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_basque_bpb
+dataset_name: Basque
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_belarusian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_belarusian_bpb.yaml
new file mode 100644
index 00000000..55173014
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_belarusian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_belarusian_bpb
+dataset_name: Belarusian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_bulgarian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_bulgarian_bpb.yaml
new file mode 100644
index 00000000..40c65e68
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_bulgarian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_bulgarian_bpb
+dataset_name: Bulgarian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_croatian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_croatian_bpb.yaml
new file mode 100644
index 00000000..3ebd7109
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_croatian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_croatian_bpb
+dataset_name: Croatian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_dutch_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_dutch_bpb.yaml
new file mode 100644
index 00000000..35d37e05
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_dutch_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_dutch_bpb
+dataset_name: Dutch
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_estonian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_estonian_bpb.yaml
new file mode 100644
index 00000000..ce298876
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_estonian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_estonian_bpb
+dataset_name: Estonian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_finnish_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_finnish_bpb.yaml
new file mode 100644
index 00000000..669e950e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_finnish_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_finnish_bpb
+dataset_name: Finnish
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_french_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_french_bpb.yaml
new file mode 100644
index 00000000..7cee7615
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_french_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_french_bpb
+dataset_name: French
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_georgian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_georgian_bpb.yaml
new file mode 100644
index 00000000..e8c3a0cd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_georgian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_georgian_bpb
+dataset_name: Georgian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_german_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_german_bpb.yaml
new file mode 100644
index 00000000..56439977
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_german_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_german_bpb
+dataset_name: German
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_greek_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_greek_bpb.yaml
new file mode 100644
index 00000000..44d4a3bb
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_greek_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_greek_bpb
+dataset_name: Greek
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_hungarian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_hungarian_bpb.yaml
new file mode 100644
index 00000000..5c478d6f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_hungarian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_hungarian_bpb
+dataset_name: Hungarian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_italian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_italian_bpb.yaml
new file mode 100644
index 00000000..893d68fd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_italian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_italian_bpb
+dataset_name: Italian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_lithuanian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_lithuanian_bpb.yaml
new file mode 100644
index 00000000..c3748d40
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_lithuanian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_lithuanian_bpb
+dataset_name: Lithuanian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_north_macedonian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_north_macedonian_bpb.yaml
new file mode 100644
index 00000000..d9eda43a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_north_macedonian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_north_macedonian_bpb
+dataset_name: North Macedonian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_polish_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_polish_bpb.yaml
new file mode 100644
index 00000000..9d327ca8
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_polish_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_polish_bpb
+dataset_name: Polish
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_portuguese_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_portuguese_bpb.yaml
new file mode 100644
index 00000000..489047e2
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_portuguese_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_portuguese_bpb
+dataset_name: Portuguese
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_russian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_russian_bpb.yaml
new file mode 100644
index 00000000..5c7b30d6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_russian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_russian_bpb
+dataset_name: Russian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_serbian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_serbian_bpb.yaml
new file mode 100644
index 00000000..8e399b3d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_serbian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_serbian_bpb
+dataset_name: Serbian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_spanish_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_spanish_bpb.yaml
new file mode 100644
index 00000000..61f3e531
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_spanish_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_spanish_bpb
+dataset_name: Spanish
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_turkish_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_turkish_bpb.yaml
new file mode 100644
index 00000000..2e8d02ba
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_turkish_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_turkish_bpb
+dataset_name: Turkish
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_ukrainian_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_ukrainian_bpb.yaml
new file mode 100644
index 00000000..d4a63944
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/include/include_base_44_ukrainian_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_include_bpb_template_yaml
+task: include_base_44_ukrainian_bpb
+dataset_name: Ukrainian
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_de_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_de_bpb.yaml
new file mode 100644
index 00000000..6acc1354
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_de_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_mgsm_bpb_template_yaml
+task: mgsm_direct_de_bpb
+dataset_name: de
+doc_to_text: '{% if answer is not none %}{{question+"\nAntwort:"}}{% else %}{{"Frage: "+question+"\nAntwort:"}}{% endif %}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_en_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_en_bpb.yaml
new file mode 100644
index 00000000..a3d17ac7
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_en_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_mgsm_bpb_template_yaml
+task: mgsm_direct_en_bpb
+dataset_name: en
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_es_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_es_bpb.yaml
new file mode 100644
index 00000000..72b385e6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_es_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_mgsm_bpb_template_yaml
+task: mgsm_direct_es_bpb
+dataset_name: es
+doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta:"}}{% else %}{{"Pregunta: "+question+"\nRespuesta:"}}{% endif %}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_fr_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_fr_bpb.yaml
new file mode 100644
index 00000000..a64bc4da
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mgsm/mgsm_direct_fr_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_mgsm_bpb_template_yaml
+task: mgsm_direct_fr_bpb
+dataset_name: fr
+doc_to_text: '{% if answer is not none %}{{question+"\nRéponse :"}}{% else %}{{"Question : "+question+"\nRéponse :"}}{% endif %}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_abstract_algebra_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_abstract_algebra_bpb.yaml
new file mode 100644
index 00000000..2e8f24ac
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_abstract_algebra_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_abstract_algebra_bpb
+dataset_name: abstract_algebra
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_anatomy_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_anatomy_bpb.yaml
new file mode 100644
index 00000000..6b038236
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_anatomy_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_anatomy_bpb
+dataset_name: anatomy
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_astronomy_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_astronomy_bpb.yaml
new file mode 100644
index 00000000..e8abb094
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_astronomy_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_astronomy_bpb
+dataset_name: astronomy
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_biology_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_biology_bpb.yaml
new file mode 100644
index 00000000..90f06223
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_biology_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_college_biology_bpb
+dataset_name: college_biology
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_chemistry_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_chemistry_bpb.yaml
new file mode 100644
index 00000000..9839d22f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_chemistry_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_college_chemistry_bpb
+dataset_name: college_chemistry
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_computer_science_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_computer_science_bpb.yaml
new file mode 100644
index 00000000..f95ab073
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_computer_science_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_college_computer_science_bpb
+dataset_name: college_computer_science
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_mathematics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_mathematics_bpb.yaml
new file mode 100644
index 00000000..20b76800
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_mathematics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_college_mathematics_bpb
+dataset_name: college_mathematics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_physics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_physics_bpb.yaml
new file mode 100644
index 00000000..e749fbb6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_college_physics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_college_physics_bpb
+dataset_name: college_physics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_computer_security_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_computer_security_bpb.yaml
new file mode 100644
index 00000000..a47d1a88
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_computer_security_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_computer_security_bpb
+dataset_name: computer_security
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_conceptual_physics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_conceptual_physics_bpb.yaml
new file mode 100644
index 00000000..0e2693f7
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_conceptual_physics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_conceptual_physics_bpb
+dataset_name: conceptual_physics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_electrical_engineering_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_electrical_engineering_bpb.yaml
new file mode 100644
index 00000000..ae8d26e5
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_electrical_engineering_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_electrical_engineering_bpb
+dataset_name: electrical_engineering
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_elementary_mathematics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_elementary_mathematics_bpb.yaml
new file mode 100644
index 00000000..219590ff
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_elementary_mathematics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_elementary_mathematics_bpb
+dataset_name: elementary_mathematics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_biology_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_biology_bpb.yaml
new file mode 100644
index 00000000..e174245f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_biology_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_high_school_biology_bpb
+dataset_name: high_school_biology
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_chemistry_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_chemistry_bpb.yaml
new file mode 100644
index 00000000..d395cbae
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_chemistry_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_high_school_chemistry_bpb
+dataset_name: high_school_chemistry
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_computer_science_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_computer_science_bpb.yaml
new file mode 100644
index 00000000..44353b47
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_computer_science_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_high_school_computer_science_bpb
+dataset_name: high_school_computer_science
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_mathematics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_mathematics_bpb.yaml
new file mode 100644
index 00000000..84af7190
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_mathematics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_high_school_mathematics_bpb
+dataset_name: high_school_mathematics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_physics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_physics_bpb.yaml
new file mode 100644
index 00000000..baff24ed
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_physics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_high_school_physics_bpb
+dataset_name: high_school_physics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_statistics_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_statistics_bpb.yaml
new file mode 100644
index 00000000..cad9b14f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_high_school_statistics_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_high_school_statistics_bpb
+dataset_name: high_school_statistics
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_machine_learning_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_machine_learning_bpb.yaml
new file mode 100644
index 00000000..e29b5470
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/mmlu_machine_learning_bpb.yaml
@@ -0,0 +1,3 @@
+include: _mmlu_stem_bpb_template_yaml
+task: mmlu_machine_learning_bpb
+dataset_name: machine_learning
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/openbookqa_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/openbookqa_bpb.yaml
new file mode 100644
index 00000000..b372f1d6
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/openbookqa_bpb.yaml
@@ -0,0 +1,11 @@
+include: _bpb_template_yaml
+task: openbookqa_bpb
+dataset_path: allenai/openbookqa
+dataset_name: main
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: question_stem
+doc_to_target: "{{choices.label.index(answerKey.lstrip())}}"
+doc_to_choice: "{{choices.text}}"
+process_results: !function utils.process_results_openbookqa
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/piqa_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/piqa_bpb.yaml
new file mode 100644
index 00000000..36152c1b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/piqa_bpb.yaml
@@ -0,0 +1,11 @@
+include: _bpb_template_yaml
+task: piqa_bpb
+dataset_path: baber/piqa
+dataset_name: null
+training_split: train
+validation_split: validation
+test_split: null
+doc_to_text: "Question: {{goal}}\nAnswer:"
+doc_to_target: label
+doc_to_choice: "{{[sol1, sol2]}}"
+process_results: !function utils.process_results_piqa
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/social_iqa_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/social_iqa_bpb.yaml
new file mode 100644
index 00000000..17b0b5ed
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/social_iqa_bpb.yaml
@@ -0,0 +1,10 @@
+include: _bpb_template_yaml
+task: social_iqa_bpb
+dataset_path: allenai/social_i_qa
+dataset_name: null
+training_split: train
+validation_split: validation
+doc_to_text: "Question: {{context}} {{question}}\nAnswer:"
+doc_to_target: "{{label|int - 1}}"
+doc_to_choice: "{{[answerA, answerB, answerC]}}"
+process_results: !function utils.process_results_social_iqa
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/utils.py b/oellm/resources/custom_lm_eval_tasks/bpb/utils.py
new file mode 100644
index 00000000..8f520914
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/utils.py
@@ -0,0 +1,411 @@
+"""Bits-per-byte (BPB) scoring for multiple-choice lm-eval-harness tasks.
+
+Why this file exists
+--------------------
+lm-eval-harness only wires ``bits_per_byte`` into ``loglikelihood_rolling``
+tasks (wikitext-style perplexity). For ``multiple_choice`` tasks the built-in
+``ConfigurableTask.process_results`` emits a fixed set of keys
+(``acc``/``acc_norm``/``acc_bytes``/``f1``/``mcc``/``exact_match``/``brier_score``),
+so adding ``bits_per_byte`` to a task's ``metric_list`` alone does nothing.
+
+The tasks in this directory therefore override ``process_results`` with the
+functions below. They compute, per document:
+
+    (loglikelihood_of_gold_continuation, byte_length_of_gold_continuation)
+
+which is exactly the ``(a, b)`` pair shape that lm-eval's already-registered
+``bits_per_byte`` *aggregation* consumes:
+
+    bits_per_byte = -sum(a) / sum(b) / ln(2)
+
+i.e. a corpus-level, byte-weighted average, not a mean of per-doc ratios. That
+is the standard BPB definition used by DCLM/OLMo/Olmix, and it is what makes
+BPB a smooth fitness signal for data-mixture regression at swarm scale, where
+accuracy on 30M-parameter models is close to chance and very noisy.
+
+Byte-length convention
+----------------------
+Bytes are counted on the continuation string exactly as returned by the task's
+``doc_to_choice`` (UTF-8), *without* the ``target_delimiter`` (the leading
+space lm-eval inserts between context and continuation). This matches lm-eval's
+own ``acc_bytes`` convention. Two notes:
+
+* Absolute BPB values shift slightly under a different convention, but the
+  *ranking* of data mixtures -- the only thing the Olmix regressor consumes --
+  is unaffected, as long as every run in the swarm uses the same convention.
+* Do not compare these numbers against BPB from another codebase without first
+  checking that codebase's byte convention.
+
+Adding a task
+-------------
+Write a ``process_results_<task>(doc, results)`` that resolves the document's
+choice strings and gold index, then returns ``_score(results, choices, gold)``.
+``results`` is the list of ``(loglikelihood, is_greedy)`` pairs lm-eval
+produces, one per choice, in ``doc_to_choice`` order.
+
+Single-reference tasks (FLORES translation, MGSM answers) use
+``output_type: loglikelihood`` instead and return
+``_score_continuation(results, text)``. lm-eval does not insert
+``target_delimiter`` for ``loglikelihood`` requests, so their ``doc_to_target``
+prepends the space itself and ``text`` is the reference *without* it, keeping
+the byte convention above.
+"""
+
+import re
+
+# --------------------------------------------------------------------------
+# Core scoring
+# --------------------------------------------------------------------------
+
+
+def _loglikelihoods(results):
+    """Extract per-choice loglikelihoods from lm-eval's result payload."""
+    lls = []
+    for r in results:
+        # multiple_choice requests come back as (loglikelihood, is_greedy)
+        lls.append(float(r[0]) if isinstance(r, (tuple, list)) else float(r))
+    return lls
+
+
+def _byte_len(text):
+    """UTF-8 byte length, floored at 1 so an empty choice can't divide by zero."""
+    return max(len(str(text).encode("utf-8")), 1)
+
+
+def _resolve_gold(gold, choices):
+    """Normalise a gold label to a valid index into ``choices``, or None."""
+    if isinstance(gold, (list, tuple)):
+        # Multi-target tasks: score against the first valid gold.
+        for g in gold:
+            resolved = _resolve_gold(g, choices)
+            if resolved is not None:
+                return resolved
+        return None
+    if isinstance(gold, str):
+        if gold in choices:
+            return choices.index(gold)
+        try:
+            gold = int(gold)
+        except (TypeError, ValueError):
+            return None
+    if isinstance(gold, int) and 0 <= gold < len(choices):
+        return gold
+    return None
+
+
+def _score(results, choices, gold, continuation=None):
+    """Return the metric dict for one document.
+
+    Args:
+        results: list of (loglikelihood, is_greedy) pairs, one per choice.
+        choices: the continuation strings, in ``doc_to_choice`` order.
+        gold: index (or label) of the correct choice.
+        continuation: for ``multiple_input`` tasks (e.g. winogrande) the choices
+            are *contexts* and the scored continuation is shared; pass it here
+            so bytes are counted on the continuation rather than on the context.
+
+    Returns:
+        dict with ``bits_per_byte`` as an (loglikelihood, n_bytes) pair plus
+        ``acc``/``acc_norm`` for free, since we already have every choice's
+        loglikelihood.
+    """
+    lls = _loglikelihoods(results)
+    choices = [str(c) for c in choices]
+    gold_idx = _resolve_gold(gold, choices)
+
+    if gold_idx is None or len(lls) != len(choices):
+        # Malformed doc: contribute nothing rather than poisoning the corpus
+        # aggregate with a bogus (ll, bytes) pair.
+        return {"acc": 0.0, "acc_norm": 0.0, "bits_per_byte": (0.0, 1)}
+
+    scored_text = continuation if continuation is not None else choices[gold_idx]
+    gold_bytes = _byte_len(scored_text)
+
+    char_lens = [max(len(c), 1) for c in choices]
+    pred = max(range(len(lls)), key=lambda i: lls[i])
+    pred_norm = max(range(len(lls)), key=lambda i: lls[i] / char_lens[i])
+
+    return {
+        "acc": 1.0 if pred == gold_idx else 0.0,
+        "acc_norm": 1.0 if pred_norm == gold_idx else 0.0,
+        # Consumed by lm-eval's registered "bits_per_byte" aggregation.
+        "bits_per_byte": (lls[gold_idx], gold_bytes),
+    }
+
+
+def _score_continuation(results, continuation):
+    """Return the metric dict for one ``loglikelihood`` (single-reference) doc.
+
+    Args:
+        results: one-element list holding the reference's (loglikelihood, is_greedy).
+        continuation: the reference text, without the leading space that the
+            task's ``doc_to_target`` prepends.
+    """
+    return {"bits_per_byte": (_loglikelihoods(results)[0], _byte_len(continuation))}
+
+
+# --------------------------------------------------------------------------
+# ARC (easy + challenge) and OpenBookQA share the AI2 label/answerKey schema
+# --------------------------------------------------------------------------
+
+
+def _ai2_choices_and_gold(doc, answer_key="answerKey"):
+    choices = list(doc["choices"]["text"])
+    labels = list(doc["choices"]["label"])
+    key = str(doc[answer_key]).strip()
+    gold = labels.index(key) if key in labels else _resolve_gold(key, choices)
+    return choices, gold
+
+
+def process_results_arc(doc, results):
+    choices, gold = _ai2_choices_and_gold(doc)
+    return _score(results, choices, gold)
+
+
+def process_results_openbookqa(doc, results):
+    choices, gold = _ai2_choices_and_gold(doc)
+    return _score(results, choices, gold)
+
+
+# --------------------------------------------------------------------------
+# HellaSwag
+# --------------------------------------------------------------------------
+
+
+def _hellaswag_preprocess(text):
+    text = text.strip()
+    # Brackets are artifacts of the WikiHow portion of HellaSwag.
+    text = text.replace(" [title]", ". ")
+    text = re.sub("\\[.*?\\]", "", text)
+    text = text.replace("  ", " ")
+    return text
+
+
+def process_docs_hellaswag(dataset):
+    def _process_doc(doc):
+        ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize()
+        return {
+            "query": _hellaswag_preprocess(doc["activity_label"] + ": " + ctx),
+            "choices": [_hellaswag_preprocess(ending) for ending in doc["endings"]],
+            "gold": int(doc["label"]),
+        }
+
+    return dataset.map(_process_doc)
+
+
+def process_results_hellaswag(doc, results):
+    gold = doc["gold"] if "gold" in doc else doc["label"]
+    return _score(results, list(doc["choices"]), gold)
+
+
+# --------------------------------------------------------------------------
+# PIQA / BoolQ / Social IQa
+# --------------------------------------------------------------------------
+
+
+def process_results_piqa(doc, results):
+    return _score(results, [doc["sol1"], doc["sol2"]], int(doc["label"]))
+
+
+def process_results_boolq(doc, results):
+    return _score(results, ["no", "yes"], int(doc["label"]))
+
+
+def process_results_social_iqa(doc, results):
+    choices = [doc["answerA"], doc["answerB"], doc["answerC"]]
+    # social_i_qa labels are 1-indexed strings.
+    return _score(results, choices, int(str(doc["label"]).strip()) - 1)
+
+
+# --------------------------------------------------------------------------
+# COPA (reimplemented here so this directory has no cross-task imports)
+# --------------------------------------------------------------------------
+
+
+def _copa_convert_choice(choice):
+    return choice[0].lower() + choice[1:]
+
+
+def doc_to_text_copa(doc):
+    connector = {"cause": "because", "effect": "therefore"}[doc["question"]]
+    # Drop the trailing period before appending the connector.
+    return doc["premise"].strip()[:-1] + f" {connector}"
+
+
+def doc_to_choice_copa(doc):
+    return [
+        " " + _copa_convert_choice(doc["choice1"]),
+        " " + _copa_convert_choice(doc["choice2"]),
+    ]
+
+
+def doc_to_target_copa(doc):
+    correct = doc["choice1"] if doc["label"] == 0 else doc["choice2"]
+    return " " + _copa_convert_choice(correct)
+
+
+def process_results_copa(doc, results):
+    return _score(results, doc_to_choice_copa(doc), int(doc["label"]))
+
+
+# --------------------------------------------------------------------------
+# CommonsenseQA -- cloze form
+#
+# The stock lm-eval task scores the letters "A".."E", so its BPB would measure
+# how well the model predicts a single letter, not whether it knows the answer.
+# These tasks put the answer *text* in the continuation instead.
+# --------------------------------------------------------------------------
+
+
+def process_results_commonsense_qa(doc, results):
+    choices, gold = _ai2_choices_and_gold(doc)
+    return _score(results, choices, gold)
+
+
+# --------------------------------------------------------------------------
+# MMLU -- cloze/continuation form, for the same reason as CommonsenseQA
+# --------------------------------------------------------------------------
+
+
+def process_results_mmlu(doc, results):
+    return _score(results, list(doc["choices"]), int(doc["answer"]))
+
+
+# --------------------------------------------------------------------------
+# Winogrande -- multiple_input: the choices are contexts and the continuation
+# is shared, so bytes are counted on the continuation.
+# --------------------------------------------------------------------------
+
+
+def doc_to_text_winogrande(doc):
+    return {"1": 0, "2": 1}[doc["answer"]]
+
+
+def doc_to_target_winogrande(doc):
+    idx = doc["sentence"].index("_") + 1
+    return doc["sentence"][idx:].strip()
+
+
+def doc_to_choice_winogrande(doc):
+    idx = doc["sentence"].index("_")
+    return [doc["sentence"][:idx] + opt for opt in [doc["option1"], doc["option2"]]]
+
+
+def process_results_winogrande(doc, results):
+    return _score(
+        results,
+        doc_to_choice_winogrande(doc),
+        doc_to_text_winogrande(doc),
+        continuation=doc_to_target_winogrande(doc),
+    )
+
+
+# ==========================================================================
+# Multilingual tasks (the oellm-multilingual-bpb super group)
+#
+# Task YAMLs live in per-benchmark subdirectories and include the
+# _<benchmark>_bpb_template_yaml files next to this module. XWinograd has the
+# Winogrande schema and reuses the Winogrande functions above.
+# ==========================================================================
+
+
+# --------------------------------------------------------------------------
+# Belebele -- cloze form: the continuation is the answer text, not a letter
+# --------------------------------------------------------------------------
+
+
+def process_results_belebele(doc, results):
+    choices = [doc[f"mc_answer{i}"] for i in range(1, 5)]
+    return _score(results, choices, int(doc["correct_answer_num"]) - 1)
+
+
+# --------------------------------------------------------------------------
+# Global-MMLU / INCLUDE -- cloze form, for the same reason as MMLU
+# --------------------------------------------------------------------------
+
+
+def _exam_choices(doc):
+    return [doc["option_a"], doc["option_b"], doc["option_c"], doc["option_d"]]
+
+
+def process_results_global_mmlu(doc, results):
+    # Global-MMLU stores the gold answer as a letter.
+    gold = {"A": 0, "B": 1, "C": 2, "D": 3}.get(str(doc["answer"]).strip())
+    return _score(results, _exam_choices(doc), gold)
+
+
+def process_results_include(doc, results):
+    return _score(results, _exam_choices(doc), int(doc["answer"]))
+
+
+# --------------------------------------------------------------------------
+# XCOPA -- same prompt as the upstream lm-eval xcopa tasks. The per-language
+# cause/effect connector is in each task's doc_to_text (xcopa/*.yaml).
+# --------------------------------------------------------------------------
+
+
+def doc_to_choice_xcopa(doc):
+    return [_copa_convert_choice(doc["choice1"]), _copa_convert_choice(doc["choice2"])]
+
+
+def process_results_xcopa(doc, results):
+    return _score(results, doc_to_choice_xcopa(doc), int(doc["label"]))
+
+
+# --------------------------------------------------------------------------
+# XStoryCloze
+# --------------------------------------------------------------------------
+
+
+def process_results_xstorycloze(doc, results):
+    choices = [doc["sentence_quiz1"], doc["sentence_quiz2"]]
+    return _score(results, choices, int(doc["answer_right_ending"]) - 1)
+
+
+# --------------------------------------------------------------------------
+# MGSM -- the test split has no worked solutions, only the final number, so
+# BPB is measured on the number that follows the upstream "direct" prompt.
+# --------------------------------------------------------------------------
+
+
+def _mgsm_answer(doc):
+    answer = doc["answer_number"]
+    if isinstance(answer, float) and answer.is_integer():
+        answer = int(answer)
+    return str(answer)
+
+
+def doc_to_target_mgsm(doc):
+    return " " + _mgsm_answer(doc)
+
+
+def process_results_mgsm(doc, results):
+    return _score_continuation(results, _mgsm_answer(doc))
+
+
+# --------------------------------------------------------------------------
+# FLORES-200 -- BPB of the reference translation given the source sentence.
+# The facebook/flores pair configs (e.g. deu_Latn-eng_Latn) carry exactly two
+# sentence_<lang> columns, one of which is English.
+# --------------------------------------------------------------------------
+
+
+def _flores_non_english(doc):
+    (key,) = [k for k in doc if k.startswith("sentence_") and k != "sentence_eng_Latn"]
+    return doc[key]
+
+
+def doc_to_target_flores_xx_eng(doc):
+    return " " + doc["sentence_eng_Latn"]
+
+
+def process_results_flores_xx_eng(doc, results):
+    return _score_continuation(results, doc["sentence_eng_Latn"])
+
+
+def doc_to_target_flores_eng_xx(doc):
+    return " " + _flores_non_english(doc)
+
+
+def process_results_flores_eng_xx(doc, results):
+    return _score_continuation(results, _flores_non_english(doc))
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/winogrande_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/winogrande_bpb.yaml
new file mode 100644
index 00000000..c191c3ec
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/winogrande_bpb.yaml
@@ -0,0 +1,12 @@
+# multiple_input task: the two options are substituted into the *context* and
+# the continuation is shared, so BPB is measured on the shared continuation.
+include: _bpb_template_yaml
+task: winogrande_bpb
+dataset_path: allenai/winogrande
+dataset_name: winogrande_xl
+training_split: train
+validation_split: validation
+doc_to_text: !function utils.doc_to_text_winogrande
+doc_to_target: !function utils.doc_to_target_winogrande
+doc_to_choice: !function utils.doc_to_choice_winogrande
+process_results: !function utils.process_results_winogrande
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_et_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_et_bpb.yaml
new file mode 100644
index 00000000..5c730b1b
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_et_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_et_bpb
+dataset_name: et
+doc_to_text: '{{premise.strip()[:-1]}} {{"sest" if question == "cause" else "seetõttu"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_ht_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_ht_bpb.yaml
new file mode 100644
index 00000000..d40d8c75
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_ht_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_ht_bpb
+dataset_name: ht
+doc_to_text: '{{premise.strip()[:-1]}} {{"poukisa" if question == "cause" else "donk sa"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_id_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_id_bpb.yaml
new file mode 100644
index 00000000..cefaea6e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_id_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_id_bpb
+dataset_name: id
+doc_to_text: '{{premise.strip()[:-1]}} {{"karena" if question == "cause" else "maka"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_it_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_it_bpb.yaml
new file mode 100644
index 00000000..78e931dc
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_it_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_it_bpb
+dataset_name: it
+doc_to_text: '{{premise.strip()[:-1]}} {{"perché" if question == "cause" else "quindi"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_qu_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_qu_bpb.yaml
new file mode 100644
index 00000000..43200f3f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_qu_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_qu_bpb
+dataset_name: qu
+doc_to_text: '{{premise.strip()[:-1]}} {{"imataq" if question == "cause" else "chaymi"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_sw_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_sw_bpb.yaml
new file mode 100644
index 00000000..bc4ed78a
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_sw_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_sw_bpb
+dataset_name: sw
+doc_to_text: '{{premise.strip()[:-1]}} {{"kwa sababu" if question == "cause" else "kwa hiyo"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_ta_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_ta_bpb.yaml
new file mode 100644
index 00000000..90692a93
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_ta_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_ta_bpb
+dataset_name: ta
+doc_to_text: '{{premise.strip()[:-1]}} {{"காரணமாக" if question == "cause" else "எனவே"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_th_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_th_bpb.yaml
new file mode 100644
index 00000000..779b920f
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_th_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_th_bpb
+dataset_name: th
+doc_to_text: '{{premise.strip()[:-1]}} {{"เพราะ" if question == "cause" else "ดังนั้น"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_tr_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_tr_bpb.yaml
new file mode 100644
index 00000000..a029979e
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_tr_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_tr_bpb
+dataset_name: tr
+doc_to_text: '{{premise.strip()[:-1]}} {{"çünkü" if question == "cause" else "bu yüzden"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_vi_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_vi_bpb.yaml
new file mode 100644
index 00000000..47f0a7ec
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_vi_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_vi_bpb
+dataset_name: vi
+doc_to_text: '{{premise.strip()[:-1]}} {{"bởi vì" if question == "cause" else "vì vậy"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_zh_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_zh_bpb.yaml
new file mode 100644
index 00000000..baffa28d
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xcopa/xcopa_zh_bpb.yaml
@@ -0,0 +1,4 @@
+include: ../_xcopa_bpb_template_yaml
+task: xcopa_zh_bpb
+dataset_name: zh
+doc_to_text: '{{premise.strip()[:-1]}} {{"因为" if question == "cause" else "所以"}}'
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_ar_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_ar_bpb.yaml
new file mode 100644
index 00000000..a3ba7975
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_ar_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_ar_bpb
+dataset_name: ar
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_en_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_en_bpb.yaml
new file mode 100644
index 00000000..53909233
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_en_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_en_bpb
+dataset_name: en
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_es_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_es_bpb.yaml
new file mode 100644
index 00000000..e3128bfe
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_es_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_es_bpb
+dataset_name: es
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_eu_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_eu_bpb.yaml
new file mode 100644
index 00000000..b7a154c1
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_eu_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_eu_bpb
+dataset_name: eu
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_hi_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_hi_bpb.yaml
new file mode 100644
index 00000000..a31b631c
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_hi_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_hi_bpb
+dataset_name: hi
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_id_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_id_bpb.yaml
new file mode 100644
index 00000000..f6eb34dd
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_id_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_id_bpb
+dataset_name: id
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_my_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_my_bpb.yaml
new file mode 100644
index 00000000..965136fe
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_my_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_my_bpb
+dataset_name: my
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_ru_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_ru_bpb.yaml
new file mode 100644
index 00000000..aa499c32
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_ru_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_ru_bpb
+dataset_name: ru
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_sw_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_sw_bpb.yaml
new file mode 100644
index 00000000..7f76bca0
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_sw_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_sw_bpb
+dataset_name: sw
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_te_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_te_bpb.yaml
new file mode 100644
index 00000000..82357523
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_te_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_te_bpb
+dataset_name: te
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_zh_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_zh_bpb.yaml
new file mode 100644
index 00000000..ffc610d3
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xstorycloze/xstorycloze_zh_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xstorycloze_bpb_template_yaml
+task: xstorycloze_zh_bpb
+dataset_name: zh
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_en_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_en_bpb.yaml
new file mode 100644
index 00000000..d0140447
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_en_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xwinograd_bpb_template_yaml
+task: xwinograd_en_bpb
+dataset_name: en
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_fr_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_fr_bpb.yaml
new file mode 100644
index 00000000..de5fff36
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_fr_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xwinograd_bpb_template_yaml
+task: xwinograd_fr_bpb
+dataset_name: fr
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_jp_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_jp_bpb.yaml
new file mode 100644
index 00000000..f1d0df47
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_jp_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xwinograd_bpb_template_yaml
+task: xwinograd_jp_bpb
+dataset_name: jp
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_pt_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_pt_bpb.yaml
new file mode 100644
index 00000000..5fd81236
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_pt_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xwinograd_bpb_template_yaml
+task: xwinograd_pt_bpb
+dataset_name: pt
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_ru_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_ru_bpb.yaml
new file mode 100644
index 00000000..ba32f2fe
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_ru_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xwinograd_bpb_template_yaml
+task: xwinograd_ru_bpb
+dataset_name: ru
diff --git a/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_zh_bpb.yaml b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_zh_bpb.yaml
new file mode 100644
index 00000000..707d3385
--- /dev/null
+++ b/oellm/resources/custom_lm_eval_tasks/bpb/xwinograd/xwinograd_zh_bpb.yaml
@@ -0,0 +1,3 @@
+include: ../_xwinograd_bpb_template_yaml
+task: xwinograd_zh_bpb
+dataset_name: zh
diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml
index 21242cc2..30ff7e9b 100644
--- a/oellm/resources/task-groups.yaml
+++ b/oellm/resources/task-groups.yaml
@@ -1,3 +1,560 @@
+task_metrics:
+  # BPB tasks: the primary metric is bits_per_byte (lower is better).
+  arc_easy_bpb: bits_per_byte
+  arc_challenge_bpb: bits_per_byte
+  hellaswag_bpb: bits_per_byte
+  piqa_bpb: bits_per_byte
+  openbookqa_bpb: bits_per_byte
+  boolq_bpb: bits_per_byte
+  copa_bpb: bits_per_byte
+  commonsense_qa_bpb: bits_per_byte
+  social_iqa_bpb: bits_per_byte
+  winogrande_bpb: bits_per_byte
+  mmlu_abstract_algebra_bpb: bits_per_byte
+  mmlu_anatomy_bpb: bits_per_byte
+  mmlu_astronomy_bpb: bits_per_byte
+  mmlu_college_biology_bpb: bits_per_byte
+  mmlu_college_chemistry_bpb: bits_per_byte
+  mmlu_college_computer_science_bpb: bits_per_byte
+  mmlu_college_mathematics_bpb: bits_per_byte
+  mmlu_college_physics_bpb: bits_per_byte
+  mmlu_computer_security_bpb: bits_per_byte
+  mmlu_conceptual_physics_bpb: bits_per_byte
+  mmlu_electrical_engineering_bpb: bits_per_byte
+  mmlu_elementary_mathematics_bpb: bits_per_byte
+  mmlu_high_school_biology_bpb: bits_per_byte
+  mmlu_high_school_chemistry_bpb: bits_per_byte
+  mmlu_high_school_computer_science_bpb: bits_per_byte
+  mmlu_high_school_mathematics_bpb: bits_per_byte
+  mmlu_high_school_physics_bpb: bits_per_byte
+  mmlu_high_school_statistics_bpb: bits_per_byte
+  mmlu_machine_learning_bpb: bits_per_byte
+  # Multilingual BPB tasks (oellm-multilingual-bpb): bits_per_byte, lower is better.
+  flores200_als_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_bos_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_bul_Cyrl-eng_Latn_bpb: bits_per_byte
+  flores200_cat_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_ces_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_dan_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_deu_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_ell_Grek-eng_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-als_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-bos_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-bul_Cyrl_bpb: bits_per_byte
+  flores200_eng_Latn-cat_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-ces_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-dan_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-deu_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-ell_Grek_bpb: bits_per_byte
+  flores200_eng_Latn-est_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-eus_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-fin_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-fra_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-gle_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-glg_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-hrv_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-hun_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-isl_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-ita_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-kat_Geor_bpb: bits_per_byte
+  flores200_eng_Latn-lit_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-lvs_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-mkd_Cyrl_bpb: bits_per_byte
+  flores200_eng_Latn-mlt_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-nld_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-nob_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-pol_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-por_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-ron_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-slk_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-slv_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-spa_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-srp_Cyrl_bpb: bits_per_byte
+  flores200_eng_Latn-swe_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-tur_Latn_bpb: bits_per_byte
+  flores200_eng_Latn-ukr_Cyrl_bpb: bits_per_byte
+  flores200_est_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_eus_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_fin_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_fra_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_gle_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_glg_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_hrv_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_hun_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_isl_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_ita_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_kat_Geor-eng_Latn_bpb: bits_per_byte
+  flores200_lit_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_lvs_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_mkd_Cyrl-eng_Latn_bpb: bits_per_byte
+  flores200_mlt_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_nld_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_nob_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_pol_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_por_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_ron_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_slk_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_slv_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_spa_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_srp_Cyrl-eng_Latn_bpb: bits_per_byte
+  flores200_swe_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_tur_Latn-eng_Latn_bpb: bits_per_byte
+  flores200_ukr_Cyrl-eng_Latn_bpb: bits_per_byte
+  belebele_bul_Cyrl_bpb: bits_per_byte
+  belebele_ces_Latn_bpb: bits_per_byte
+  belebele_dan_Latn_bpb: bits_per_byte
+  belebele_deu_Latn_bpb: bits_per_byte
+  belebele_ell_Grek_bpb: bits_per_byte
+  belebele_eng_Latn_bpb: bits_per_byte
+  belebele_est_Latn_bpb: bits_per_byte
+  belebele_fin_Latn_bpb: bits_per_byte
+  belebele_fra_Latn_bpb: bits_per_byte
+  belebele_hrv_Latn_bpb: bits_per_byte
+  belebele_hun_Latn_bpb: bits_per_byte
+  belebele_ita_Latn_bpb: bits_per_byte
+  belebele_lit_Latn_bpb: bits_per_byte
+  belebele_lvs_Latn_bpb: bits_per_byte
+  belebele_mlt_Latn_bpb: bits_per_byte
+  belebele_nld_Latn_bpb: bits_per_byte
+  belebele_nob_Latn_bpb: bits_per_byte
+  belebele_pol_Latn_bpb: bits_per_byte
+  belebele_por_Latn_bpb: bits_per_byte
+  belebele_ron_Latn_bpb: bits_per_byte
+  belebele_slk_Latn_bpb: bits_per_byte
+  belebele_slv_Latn_bpb: bits_per_byte
+  belebele_spa_Latn_bpb: bits_per_byte
+  belebele_swe_Latn_bpb: bits_per_byte
+  global_mmlu_cs_bpb: bits_per_byte
+  global_mmlu_de_bpb: bits_per_byte
+  global_mmlu_el_bpb: bits_per_byte
+  global_mmlu_en_bpb: bits_per_byte
+  global_mmlu_es_bpb: bits_per_byte
+  global_mmlu_fr_bpb: bits_per_byte
+  global_mmlu_he_bpb: bits_per_byte
+  global_mmlu_it_bpb: bits_per_byte
+  global_mmlu_lt_bpb: bits_per_byte
+  global_mmlu_nl_bpb: bits_per_byte
+  global_mmlu_pl_bpb: bits_per_byte
+  global_mmlu_pt_bpb: bits_per_byte
+  global_mmlu_ro_bpb: bits_per_byte
+  global_mmlu_ru_bpb: bits_per_byte
+  global_mmlu_sr_bpb: bits_per_byte
+  global_mmlu_sv_bpb: bits_per_byte
+  global_mmlu_tr_bpb: bits_per_byte
+  global_mmlu_uk_bpb: bits_per_byte
+  mgsm_direct_de_bpb: bits_per_byte
+  mgsm_direct_en_bpb: bits_per_byte
+  mgsm_direct_es_bpb: bits_per_byte
+  mgsm_direct_fr_bpb: bits_per_byte
+  xwinograd_en_bpb: bits_per_byte
+  xwinograd_fr_bpb: bits_per_byte
+  xwinograd_jp_bpb: bits_per_byte
+  xwinograd_pt_bpb: bits_per_byte
+  xwinograd_ru_bpb: bits_per_byte
+  xwinograd_zh_bpb: bits_per_byte
+  xcopa_et_bpb: bits_per_byte
+  xcopa_ht_bpb: bits_per_byte
+  xcopa_id_bpb: bits_per_byte
+  xcopa_it_bpb: bits_per_byte
+  xcopa_qu_bpb: bits_per_byte
+  xcopa_sw_bpb: bits_per_byte
+  xcopa_ta_bpb: bits_per_byte
+  xcopa_th_bpb: bits_per_byte
+  xcopa_tr_bpb: bits_per_byte
+  xcopa_vi_bpb: bits_per_byte
+  xcopa_zh_bpb: bits_per_byte
+  xstorycloze_ar_bpb: bits_per_byte
+  xstorycloze_en_bpb: bits_per_byte
+  xstorycloze_es_bpb: bits_per_byte
+  xstorycloze_eu_bpb: bits_per_byte
+  xstorycloze_hi_bpb: bits_per_byte
+  xstorycloze_id_bpb: bits_per_byte
+  xstorycloze_my_bpb: bits_per_byte
+  xstorycloze_ru_bpb: bits_per_byte
+  xstorycloze_sw_bpb: bits_per_byte
+  xstorycloze_te_bpb: bits_per_byte
+  xstorycloze_zh_bpb: bits_per_byte
+  include_base_44_albanian_bpb: bits_per_byte
+  include_base_44_armenian_bpb: bits_per_byte
+  include_base_44_azerbaijani_bpb: bits_per_byte
+  include_base_44_basque_bpb: bits_per_byte
+  include_base_44_belarusian_bpb: bits_per_byte
+  include_base_44_bulgarian_bpb: bits_per_byte
+  include_base_44_croatian_bpb: bits_per_byte
+  include_base_44_dutch_bpb: bits_per_byte
+  include_base_44_estonian_bpb: bits_per_byte
+  include_base_44_finnish_bpb: bits_per_byte
+  include_base_44_french_bpb: bits_per_byte
+  include_base_44_georgian_bpb: bits_per_byte
+  include_base_44_german_bpb: bits_per_byte
+  include_base_44_greek_bpb: bits_per_byte
+  include_base_44_hungarian_bpb: bits_per_byte
+  include_base_44_italian_bpb: bits_per_byte
+  include_base_44_lithuanian_bpb: bits_per_byte
+  include_base_44_north_macedonian_bpb: bits_per_byte
+  include_base_44_polish_bpb: bits_per_byte
+  include_base_44_portuguese_bpb: bits_per_byte
+  include_base_44_russian_bpb: bits_per_byte
+  include_base_44_serbian_bpb: bits_per_byte
+  include_base_44_spanish_bpb: bits_per_byte
+  include_base_44_turkish_bpb: bits_per_byte
+  include_base_44_ukrainian_bpb: bits_per_byte
+  mmlu: acc
+  mmlu_prox_cs: exact_match
+  mmlu_prox_de: exact_match
+  mmlu_prox_en: exact_match
+  mmlu_prox_es: exact_match
+  mmlu_prox_fr: exact_match
+  mmlu_prox_hu: exact_match
+  mmlu_prox_it: exact_match
+  mmlu_prox_pt: exact_match
+  mmlu_prox_sr: exact_match
+  mmlu_prox_uk: exact_match
+  sib200_bul_Cyrl: acc_norm
+  sib200_hrv_Latn: acc_norm
+  sib200_ces_Latn: acc_norm
+  sib200_dan_Latn: acc_norm
+  sib200_nld_Latn: acc_norm
+  sib200_eng_Latn: acc_norm
+  sib200_est_Latn: acc_norm
+  sib200_fin_Latn: acc_norm
+  sib200_fra_Latn: acc_norm
+  sib200_deu_Latn: acc_norm
+  sib200_ell_Grek: acc_norm
+  sib200_hun_Latn: acc_norm
+  sib200_gle_Latn: acc_norm
+  sib200_ita_Latn: acc_norm
+  sib200_lvs_Latn: acc_norm
+  sib200_lit_Latn: acc_norm
+  sib200_mlt_Latn: acc_norm
+  sib200_pol_Latn: acc_norm
+  sib200_por_Latn: acc_norm
+  sib200_ron_Latn: acc_norm
+  sib200_slk_Latn: acc_norm
+  sib200_slv_Latn: acc_norm
+  sib200_spa_Latn: acc_norm
+  sib200_swe_Latn: acc_norm
+  sib200_cat_Latn: acc_norm
+  sib200_eus_Latn: acc_norm
+  sib200_glg_Latn: acc_norm
+  sib200_bos_Latn: acc_norm
+  sib200_kat_Geor: acc_norm
+  sib200_mkd_Cyrl: acc_norm
+  sib200_als_Latn: acc_norm
+  sib200_srp_Cyrl: acc_norm
+  sib200_tur_Latn: acc_norm
+  sib200_ukr_Cyrl: acc_norm
+  sib200_isl_Latn: acc_norm
+  sib200_nob_Latn: acc_norm
+  belebele_bul_Cyrl_cf: acc_norm
+  belebele_hrv_Latn_cf: acc_norm
+  belebele_ces_Latn_cf: acc_norm
+  belebele_dan_Latn_cf: acc_norm
+  belebele_nld_Latn_cf: acc_norm
+  belebele_eng_Latn_cf: acc_norm
+  belebele_est_Latn_cf: acc_norm
+  belebele_fin_Latn_cf: acc_norm
+  belebele_fra_Latn_cf: acc_norm
+  belebele_deu_Latn_cf: acc_norm
+  belebele_ell_Grek_cf: acc_norm
+  belebele_hun_Latn_cf: acc_norm
+  belebele_ita_Latn_cf: acc_norm
+  belebele_lvs_Latn_cf: acc_norm
+  belebele_lit_Latn_cf: acc_norm
+  belebele_mlt_Latn_cf: acc_norm
+  belebele_pol_Latn_cf: acc_norm
+  belebele_por_Latn_cf: acc_norm
+  belebele_ron_Latn_cf: acc_norm
+  belebele_slk_Latn_cf: acc_norm
+  belebele_slv_Latn_cf: acc_norm
+  belebele_spa_Latn_cf: acc_norm
+  belebele_swe_Latn_cf: acc_norm
+  belebele_nob_Latn_cf: acc_norm
+  belebele_eus_Latn_cf: acc_norm
+  belebele_cat_Latn_cf: acc_norm
+  xcsqa_deu_Latn: acc_norm
+  xcsqa_eng_Latn: acc_norm
+  xcsqa_spa_Latn: acc_norm
+  xcsqa_fra_Latn: acc_norm
+  xcsqa_ita_Latn: acc_norm
+  xcsqa_nld_Latn: acc_norm
+  xcsqa_pol_Latn: acc_norm
+  xcsqa_por_Latn: acc_norm
+  pawsx_deu_Latn: acc
+  pawsx_eng_Latn: acc
+  pawsx_spa_Latn: acc
+  pawsx_fra_Latn: acc
+  xnli_bul_Cyrl: acc
+  xnli_deu_Latn: acc
+  xnli_ell_Grek: acc
+  xnli_eng_Latn: acc
+  xnli_spa_Latn: acc
+  xnli_fra_Latn: acc
+  copa: acc
+  lambada_openai: acc
+  openbookqa: acc_norm
+  winogrande: acc
+  arc_challenge: acc_norm
+  arc_easy: acc_norm
+  arc_challenge_mt_bg: acc_norm
+  arc_challenge_mt_cs: acc_norm
+  arc_challenge_mt_da: acc_norm
+  arc_challenge_mt_nl: acc_norm
+  arc_challenge_mt_et: acc_norm
+  arc_challenge_mt_fi: acc_norm
+  arc_challenge_mt_fr: acc_norm
+  arc_challenge_mt_de: acc_norm
+  arc_challenge_mt_el: acc_norm
+  arc_challenge_mt_hu: acc_norm
+  arc_challenge_mt_it: acc_norm
+  arc_challenge_mt_lv: acc_norm
+  arc_challenge_mt_lt: acc_norm
+  arc_challenge_mt_pl: acc_norm
+  arc_challenge_mt_pt: acc_norm
+  arc_challenge_mt_ro: acc_norm
+  arc_challenge_mt_sk: acc_norm
+  arc_challenge_mt_sl: acc_norm
+  arc_challenge_mt_es: acc_norm
+  arc_challenge_mt_sv: acc_norm
+  arc_challenge_mt_is: acc_norm
+  arc_challenge_mt_nb: acc_norm
+  boolq: acc
+  commonsense_qa: acc
+  # global-mgsm (CohereLabs/global-mgsm): flexible-extract exact_match
+  global_mgsm_ca: exact_match
+  global_mgsm_cs: exact_match
+  global_mgsm_de: exact_match
+  global_mgsm_el: exact_match
+  global_mgsm_en: exact_match
+  global_mgsm_es: exact_match
+  global_mgsm_eu: exact_match
+  global_mgsm_fr: exact_match
+  global_mgsm_gl: exact_match
+  global_mgsm_hu: exact_match
+  global_mgsm_sr: exact_match
+  hellaswag: acc_norm
+  piqa: acc_norm
+  social_iqa: acc
+  agieval_lsat_ar: acc
+  wsc273: acc
+  bigbench_language_identification_multiple_choice: acc
+  squadv2: f1
+  coqa: f1
+  bigbench_qa_wikidata_generate_until: exact_match
+  bigbench_dyck_languages_generate_until: exact_match
+  bigbench_operators_generate_until: exact_match
+  bigbench_repeat_copy_logic_generate_until: exact_match
+  bigbench_cs_algorithms_generate_until: exact_match
+  opensubtitles_multi40_en_to_bg: bleu
+  opensubtitles_multi40_en_to_hr: bleu
+  opensubtitles_multi40_en_to_cs: bleu
+  opensubtitles_multi40_en_to_da: bleu
+  opensubtitles_multi40_en_to_nl: bleu
+  opensubtitles_multi40_en_to_et: bleu
+  opensubtitles_multi40_en_to_fi: bleu
+  opensubtitles_multi40_en_to_fr: bleu
+  opensubtitles_multi40_en_to_de: bleu
+  opensubtitles_multi40_en_to_el: bleu
+  opensubtitles_multi40_en_to_hu: bleu
+  opensubtitles_multi40_en_to_it: bleu
+  opensubtitles_multi40_en_to_lv: bleu
+  opensubtitles_multi40_en_to_lt: bleu
+  opensubtitles_multi40_en_to_pl: bleu
+  opensubtitles_multi40_en_to_pt: bleu
+  opensubtitles_multi40_en_to_ro: bleu
+  opensubtitles_multi40_en_to_sk: bleu
+  opensubtitles_multi40_en_to_sl: bleu
+  opensubtitles_multi40_en_to_es: bleu
+  opensubtitles_multi40_en_to_sv: bleu
+  opensubtitles_multi40_en_to_sr: bleu
+  opensubtitles_multi40_en_to_tr: bleu
+  opensubtitles_multi40_en_to_uk: bleu
+  opensubtitles_multi40_en_to_no: bleu
+  opensubtitles_multi40_bg_to_en: bleu
+  opensubtitles_multi40_hr_to_en: bleu
+  opensubtitles_multi40_cs_to_en: bleu
+  opensubtitles_multi40_da_to_en: bleu
+  opensubtitles_multi40_nl_to_en: bleu
+  opensubtitles_multi40_et_to_en: bleu
+  opensubtitles_multi40_fi_to_en: bleu
+  opensubtitles_multi40_fr_to_en: bleu
+  opensubtitles_multi40_de_to_en: bleu
+  opensubtitles_multi40_el_to_en: bleu
+  opensubtitles_multi40_hu_to_en: bleu
+  opensubtitles_multi40_it_to_en: bleu
+  opensubtitles_multi40_lv_to_en: bleu
+  opensubtitles_multi40_lt_to_en: bleu
+  opensubtitles_multi40_pl_to_en: bleu
+  opensubtitles_multi40_pt_to_en: bleu
+  opensubtitles_multi40_ro_to_en: bleu
+  opensubtitles_multi40_sk_to_en: bleu
+  opensubtitles_multi40_sl_to_en: bleu
+  opensubtitles_multi40_es_to_en: bleu
+  opensubtitles_multi40_sv_to_en: bleu
+  opensubtitles_multi40_sr_to_en: bleu
+  opensubtitles_multi40_tr_to_en: bleu
+  opensubtitles_multi40_uk_to_en: bleu
+  opensubtitles_multi40_no_to_en: bleu
+  # flores200 (lighteval) translation: chrf++ is the primary metric
+  flores200:bul_Cyrl-eng_Latn: chrf++
+  flores200:hrv_Latn-eng_Latn: chrf++
+  flores200:ces_Latn-eng_Latn: chrf++
+  flores200:dan_Latn-eng_Latn: chrf++
+  flores200:nld_Latn-eng_Latn: chrf++
+  flores200:est_Latn-eng_Latn: chrf++
+  flores200:fin_Latn-eng_Latn: chrf++
+  flores200:fra_Latn-eng_Latn: chrf++
+  flores200:deu_Latn-eng_Latn: chrf++
+  flores200:ell_Grek-eng_Latn: chrf++
+  flores200:hun_Latn-eng_Latn: chrf++
+  flores200:gle_Latn-eng_Latn: chrf++
+  flores200:ita_Latn-eng_Latn: chrf++
+  flores200:lvs_Latn-eng_Latn: chrf++
+  flores200:lit_Latn-eng_Latn: chrf++
+  flores200:mlt_Latn-eng_Latn: chrf++
+  flores200:pol_Latn-eng_Latn: chrf++
+  flores200:por_Latn-eng_Latn: chrf++
+  flores200:ron_Latn-eng_Latn: chrf++
+  flores200:slk_Latn-eng_Latn: chrf++
+  flores200:slv_Latn-eng_Latn: chrf++
+  flores200:spa_Latn-eng_Latn: chrf++
+  flores200:swe_Latn-eng_Latn: chrf++
+  flores200:cat_Latn-eng_Latn: chrf++
+  flores200:eus_Latn-eng_Latn: chrf++
+  flores200:glg_Latn-eng_Latn: chrf++
+  flores200:bos_Latn-eng_Latn: chrf++
+  flores200:kat_Geor-eng_Latn: chrf++
+  flores200:mkd_Cyrl-eng_Latn: chrf++
+  flores200:als_Latn-eng_Latn: chrf++
+  flores200:srp_Cyrl-eng_Latn: chrf++
+  flores200:tur_Latn-eng_Latn: chrf++
+  flores200:ukr_Cyrl-eng_Latn: chrf++
+  flores200:isl_Latn-eng_Latn: chrf++
+  flores200:nob_Latn-eng_Latn: chrf++
+  flores200:eng_Latn-bul_Cyrl: chrf++
+  flores200:eng_Latn-hrv_Latn: chrf++
+  flores200:eng_Latn-ces_Latn: chrf++
+  flores200:eng_Latn-dan_Latn: chrf++
+  flores200:eng_Latn-nld_Latn: chrf++
+  flores200:eng_Latn-est_Latn: chrf++
+  flores200:eng_Latn-fin_Latn: chrf++
+  flores200:eng_Latn-fra_Latn: chrf++
+  flores200:eng_Latn-deu_Latn: chrf++
+  flores200:eng_Latn-ell_Grek: chrf++
+  flores200:eng_Latn-hun_Latn: chrf++
+  flores200:eng_Latn-gle_Latn: chrf++
+  flores200:eng_Latn-ita_Latn: chrf++
+  flores200:eng_Latn-lvs_Latn: chrf++
+  flores200:eng_Latn-lit_Latn: chrf++
+  flores200:eng_Latn-mlt_Latn: chrf++
+  flores200:eng_Latn-pol_Latn: chrf++
+  flores200:eng_Latn-por_Latn: chrf++
+  flores200:eng_Latn-ron_Latn: chrf++
+  flores200:eng_Latn-slk_Latn: chrf++
+  flores200:eng_Latn-slv_Latn: chrf++
+  flores200:eng_Latn-spa_Latn: chrf++
+  flores200:eng_Latn-swe_Latn: chrf++
+  flores200:eng_Latn-cat_Latn: chrf++
+  flores200:eng_Latn-eus_Latn: chrf++
+  flores200:eng_Latn-glg_Latn: chrf++
+  flores200:eng_Latn-bos_Latn: chrf++
+  flores200:eng_Latn-kat_Geor: chrf++
+  flores200:eng_Latn-mkd_Cyrl: chrf++
+  flores200:eng_Latn-als_Latn: chrf++
+  flores200:eng_Latn-srp_Cyrl: chrf++
+  flores200:eng_Latn-tur_Latn: chrf++
+  flores200:eng_Latn-ukr_Cyrl: chrf++
+  flores200:eng_Latn-isl_Latn: chrf++
+  flores200:eng_Latn-nob_Latn: chrf++
+  global_piqa_completions_als_latn: acc_norm
+  global_piqa_completions_bos_latn: acc_norm
+  global_piqa_completions_bul_cyrl: acc_norm
+  global_piqa_completions_cat_latn: acc_norm
+  global_piqa_completions_ces_latn: acc_norm
+  global_piqa_completions_deu_latn: acc_norm
+  global_piqa_completions_ekk_latn: acc_norm
+  global_piqa_completions_ell_grek: acc_norm
+  global_piqa_completions_eng_latn: acc_norm
+  global_piqa_completions_fin_latn: acc_norm
+  global_piqa_completions_fra_latn_fran: acc_norm
+  global_piqa_completions_glg_latn: acc_norm
+  global_piqa_completions_hrv_latn: acc_norm
+  global_piqa_completions_hun_latn: acc_norm
+  global_piqa_completions_isl_latn: acc_norm
+  global_piqa_completions_ita_latn: acc_norm
+  global_piqa_completions_kat_geor: acc_norm
+  global_piqa_completions_lit_latn: acc_norm
+  global_piqa_completions_mkd_cyrl: acc_norm
+  global_piqa_completions_nld_latn: acc_norm
+  global_piqa_completions_nno_latn: acc_norm
+  global_piqa_completions_nob_latn: acc_norm
+  global_piqa_completions_pol_latn: acc_norm
+  global_piqa_completions_por_latn_port: acc_norm
+  global_piqa_completions_ron_latn: acc_norm
+  global_piqa_completions_slk_latn: acc_norm
+  global_piqa_completions_slv_latn: acc_norm
+  global_piqa_completions_spa_latn_spai: acc_norm
+  global_piqa_completions_srp_cyrl: acc_norm
+  global_piqa_completions_swe_latn: acc_norm
+  global_piqa_completions_tur_latn: acc_norm
+  global_piqa_completions_ukr_cyrl: acc_norm
+  global_piqa_prompted_als_latn: exact_match
+  global_piqa_prompted_bos_latn: exact_match
+  global_piqa_prompted_bul_cyrl: exact_match
+  global_piqa_prompted_cat_latn: exact_match
+  global_piqa_prompted_ces_latn: exact_match
+  global_piqa_prompted_deu_latn: exact_match
+  global_piqa_prompted_ekk_latn: exact_match
+  global_piqa_prompted_ell_grek: exact_match
+  global_piqa_prompted_eng_latn: exact_match
+  global_piqa_prompted_fin_latn: exact_match
+  global_piqa_prompted_fra_latn_fran: exact_match
+  global_piqa_prompted_glg_latn: exact_match
+  global_piqa_prompted_hrv_latn: exact_match
+  global_piqa_prompted_hun_latn: exact_match
+  global_piqa_prompted_isl_latn: exact_match
+  global_piqa_prompted_ita_latn: exact_match
+  global_piqa_prompted_kat_geor: exact_match
+  global_piqa_prompted_lit_latn: exact_match
+  global_piqa_prompted_mkd_cyrl: exact_match
+  global_piqa_prompted_nld_latn: exact_match
+  global_piqa_prompted_nno_latn: exact_match
+  global_piqa_prompted_nob_latn: exact_match
+  global_piqa_prompted_pol_latn: exact_match
+  global_piqa_prompted_por_latn_port: exact_match
+  global_piqa_prompted_ron_latn: exact_match
+  global_piqa_prompted_slk_latn: exact_match
+  global_piqa_prompted_slv_latn: exact_match
+  global_piqa_prompted_spa_latn_spai: exact_match
+  global_piqa_prompted_srp_cyrl: exact_match
+  global_piqa_prompted_swe_latn: exact_match
+  global_piqa_prompted_tur_latn: exact_match
+  global_piqa_prompted_ukr_cyrl: exact_match
+  polymath_de_low: exact_match
+  polymath_de_medium: exact_match
+  polymath_de_high: exact_match
+  polymath_de_top: exact_match
+  polymath_en_low: exact_match
+  polymath_en_medium: exact_match
+  polymath_en_high: exact_match
+  polymath_en_top: exact_match
+  polymath_es_low: exact_match
+  polymath_es_medium: exact_match
+  polymath_es_high: exact_match
+  polymath_es_top: exact_match
+  polymath_fr_low: exact_match
+  polymath_fr_medium: exact_match
+  polymath_fr_high: exact_match
+  polymath_fr_top: exact_match
+  polymath_it_low: exact_match
+  polymath_it_medium: exact_match
+  polymath_it_high: exact_match
+  polymath_it_top: exact_match
+  polymath_pt_low: exact_match
+  polymath_pt_medium: exact_match
+  polymath_pt_high: exact_match
+  polymath_pt_top: exact_match
+
 task_groups:
   sib200-eu:
     description: "SIB-200 European language topic classification tasks (0-shot)"
@@ -588,6 +1145,254 @@ task_groups:
       - task: "polymath_{lang}_top"
         subset: "{lang}"
 
+  # ------------------------------------------------------------------
+  # BPB (bits-per-byte) groups.
+  #
+  # These score the *gold* continuation's loglikelihood normalised by its
+  # UTF-8 byte length, instead of argmax accuracy. At swarm scale (30M-150M
+  # params, a few B tokens) accuracy on these benchmarks sits near chance and
+  # is dominated by noise, which makes it a poor regression target; BPB is
+  # continuous, strictly decreasing with model quality, and is what
+  # DCLM/OLMo/Olmix fit data-mixture regressors on.
+  #
+  # Task definitions live in custom_lm_eval_tasks/bpb/ and are picked up via
+  # the default --include_path.
+  # ------------------------------------------------------------------
+  bpb-core:
+    description: "Bits-per-byte on the DCLM/Olmix core multiple-choice benchmarks (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    tasks:
+      - task: arc_easy_bpb
+        dataset: allenai/ai2_arc
+        subset: ARC-Easy
+      - task: arc_challenge_bpb
+        dataset: allenai/ai2_arc
+        subset: ARC-Challenge
+      - task: hellaswag_bpb
+        dataset: Rowan/hellaswag
+      - task: piqa_bpb
+        dataset: baber/piqa
+      - task: openbookqa_bpb
+        dataset: allenai/openbookqa
+        subset: main
+      - task: boolq_bpb
+        dataset: aps/super_glue
+        subset: boolq
+      - task: copa_bpb
+        dataset: aps/super_glue
+        subset: copa
+      - task: commonsense_qa_bpb
+        dataset: tau/commonsense_qa
+      - task: social_iqa_bpb
+        dataset: allenai/social_i_qa
+      - task: winogrande_bpb
+        dataset: allenai/winogrande
+        subset: winogrande_xl
+
+  bpb-mmlu-stem:
+    description: "Bits-per-byte on MMLU STEM subjects, cloze form (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: cais/mmlu
+    tasks:
+      - task: mmlu_abstract_algebra_bpb
+        subset: abstract_algebra
+      - task: mmlu_anatomy_bpb
+        subset: anatomy
+      - task: mmlu_astronomy_bpb
+        subset: astronomy
+      - task: mmlu_college_biology_bpb
+        subset: college_biology
+      - task: mmlu_college_chemistry_bpb
+        subset: college_chemistry
+      - task: mmlu_college_computer_science_bpb
+        subset: college_computer_science
+      - task: mmlu_college_mathematics_bpb
+        subset: college_mathematics
+      - task: mmlu_college_physics_bpb
+        subset: college_physics
+      - task: mmlu_computer_security_bpb
+        subset: computer_security
+      - task: mmlu_conceptual_physics_bpb
+        subset: conceptual_physics
+      - task: mmlu_electrical_engineering_bpb
+        subset: electrical_engineering
+      - task: mmlu_elementary_mathematics_bpb
+        subset: elementary_mathematics
+      - task: mmlu_high_school_biology_bpb
+        subset: high_school_biology
+      - task: mmlu_high_school_chemistry_bpb
+        subset: high_school_chemistry
+      - task: mmlu_high_school_computer_science_bpb
+        subset: high_school_computer_science
+      - task: mmlu_high_school_mathematics_bpb
+        subset: high_school_mathematics
+      - task: mmlu_high_school_physics_bpb
+        subset: high_school_physics
+      - task: mmlu_high_school_statistics_bpb
+        subset: high_school_statistics
+      - task: mmlu_machine_learning_bpb
+        subset: machine_learning
+
+  # ------------------------------------------------------------------
+  # Multilingual BPB groups: the BPB counterparts of the oellm-multilingual
+  # super group, with the same conventions as the English BPB groups above
+  # (0-shot; multiple-choice exams in cloze form). FLORES and MGSM score a
+  # single reference (translation / final answer number) instead of a choice.
+  # Task definitions live in custom_lm_eval_tasks/bpb/<benchmark>/.
+  # ------------------------------------------------------------------
+  flores-200-eu-to-eng-bpb:
+    description: "Bits-per-byte of FLORES-200 EU->English reference translations (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: facebook/flores
+    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, est_Latn,
+                  fin_Latn, fra_Latn, deu_Latn, ell_Grek, hun_Latn, gle_Latn,
+                  ita_Latn, lvs_Latn, lit_Latn, mlt_Latn, pol_Latn, por_Latn,
+                  ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, cat_Latn,
+                  eus_Latn, glg_Latn, bos_Latn, kat_Geor, mkd_Cyrl, als_Latn,
+                  srp_Cyrl, tur_Latn, ukr_Cyrl, isl_Latn, nob_Latn]
+    tasks:
+      - task: "flores200_{lang}-eng_Latn_bpb"
+        subset: "{lang}-eng_Latn"
+
+  flores-200-eng-to-eu-bpb:
+    description: "Bits-per-byte of FLORES-200 English->EU reference translations (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: facebook/flores
+    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, est_Latn,
+                  fin_Latn, fra_Latn, deu_Latn, ell_Grek, hun_Latn, gle_Latn,
+                  ita_Latn, lvs_Latn, lit_Latn, mlt_Latn, pol_Latn, por_Latn,
+                  ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, cat_Latn,
+                  eus_Latn, glg_Latn, bos_Latn, kat_Geor, mkd_Cyrl, als_Latn,
+                  srp_Cyrl, tur_Latn, ukr_Cyrl, isl_Latn, nob_Latn]
+    tasks:
+      - task: "flores200_eng_Latn-{lang}_bpb"
+        subset: "eng_Latn-{lang}"
+
+  belebele-eu-bpb:
+    description: "Bits-per-byte on Belebele European languages, cloze form (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: facebook/belebele
+    valid_langs: [bul_Cyrl, hrv_Latn, ces_Latn, dan_Latn, nld_Latn, eng_Latn, est_Latn, fin_Latn,
+                  fra_Latn, deu_Latn, ell_Grek, hun_Latn, ita_Latn, lvs_Latn, lit_Latn, mlt_Latn,
+                  pol_Latn, por_Latn, ron_Latn, slk_Latn, slv_Latn, spa_Latn, swe_Latn, nob_Latn]
+    tasks:
+      - task: "belebele_{lang}_bpb"
+        subset: "{lang}"
+
+  global-mmlu-eu-bpb:
+    description: "Bits-per-byte on Global-MMLU (all subjects per language), cloze form (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: CohereForAI/Global-MMLU
+    valid_langs: [cs, de, el, en, es, fr, it, lt, nl, pl, pt, ro, ru, sr, sv, tr, uk, he]
+    tasks:
+      - task: "global_mmlu_{lang}_bpb"
+        subset: "{lang}"
+
+  mgsm-eu-bpb:
+    description: "Bits-per-byte of the MGSM final answer after the direct prompt (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: jbross-ibm-research/mgsm
+    valid_langs: [en, de, es, fr]
+    tasks:
+      - task: "mgsm_direct_{lang}_bpb"
+        subset: "{lang}"
+
+  # XWinograd, XCOPA and XStoryCloze together mirror generic-multilingual, split
+  # per benchmark so each task carries its language.
+  xwinograd-bpb:
+    description: "Bits-per-byte on XWinograd (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: Muennighoff/xwinograd
+    valid_langs: [en, fr, jp, pt, ru, zh]
+    tasks:
+      - task: "xwinograd_{lang}_bpb"
+        subset: "{lang}"
+
+  xcopa-bpb:
+    description: "Bits-per-byte on XCOPA (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: cambridgeltl/xcopa
+    valid_langs: [et, ht, id, it, qu, sw, ta, th, tr, vi, zh]
+    tasks:
+      - task: "xcopa_{lang}_bpb"
+        subset: "{lang}"
+
+  xstorycloze-bpb:
+    description: "Bits-per-byte on XStoryCloze (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: juletxara/xstory_cloze
+    valid_langs: [ar, en, es, eu, hi, id, my, ru, sw, te, zh]
+    tasks:
+      - task: "xstorycloze_{lang}_bpb"
+        subset: "{lang}"
+
+  include-bpb:
+    description: "Bits-per-byte on INCLUDE (all domains per language), cloze form (0-shot)"
+    suite: lm-eval-harness
+    n_shots: [0]
+    dataset: CohereForAI/include-base-44
+    tasks:
+      - task: include_base_44_albanian_bpb
+        subset: Albanian
+      - task: include_base_44_armenian_bpb
+        subset: Armenian
+      - task: include_base_44_azerbaijani_bpb
+        subset: Azerbaijani
+      - task: include_base_44_basque_bpb
+        subset: Basque
+      - task: include_base_44_belarusian_bpb
+        subset: Belarusian
+      - task: include_base_44_bulgarian_bpb
+        subset: Bulgarian
+      - task: include_base_44_croatian_bpb
+        subset: Croatian
+      - task: include_base_44_dutch_bpb
+        subset: Dutch
+      - task: include_base_44_estonian_bpb
+        subset: Estonian
+      - task: include_base_44_finnish_bpb
+        subset: Finnish
+      - task: include_base_44_french_bpb
+        subset: French
+      - task: include_base_44_georgian_bpb
+        subset: Georgian
+      - task: include_base_44_german_bpb
+        subset: German
+      - task: include_base_44_greek_bpb
+        subset: Greek
+      - task: include_base_44_hungarian_bpb
+        subset: Hungarian
+      - task: include_base_44_italian_bpb
+        subset: Italian
+      - task: include_base_44_lithuanian_bpb
+        subset: Lithuanian
+      - task: include_base_44_north_macedonian_bpb
+        subset: North Macedonian
+      - task: include_base_44_polish_bpb
+        subset: Polish
+      - task: include_base_44_portuguese_bpb
+        subset: Portuguese
+      - task: include_base_44_russian_bpb
+        subset: Russian
+      - task: include_base_44_serbian_bpb
+        subset: Serbian
+      - task: include_base_44_spanish_bpb
+        subset: Spanish
+      - task: include_base_44_turkish_bpb
+        subset: Turkish
+      - task: include_base_44_ukrainian_bpb
+        subset: Ukrainian
+
 super_groups:
   oellm-multilingual:
     description: "Combined Belebele EU set plus multilingual benchmarks"
@@ -599,6 +1404,25 @@ super_groups:
       - task: mgsm-eu
       - task: generic-multilingual
       - task: include
+
+  bpb-all:
+    description: "All BPB benchmarks: core multiple-choice plus MMLU STEM"
+    task_groups:
+      - task: bpb-core
+      - task: bpb-mmlu-stem
+
+  oellm-multilingual-bpb:
+    description: "Bits-per-byte counterparts of the oellm-multilingual benchmarks"
+    task_groups:
+      - task: flores-200-eu-to-eng-bpb
+      - task: flores-200-eng-to-eu-bpb
+      - task: belebele-eu-bpb
+      - task: global-mmlu-eu-bpb
+      - task: mgsm-eu-bpb
+      - task: xwinograd-bpb
+      - task: xcopa-bpb
+      - task: xstorycloze-bpb
+      - task: include-bpb
       - task: multiblimp-eu
       - task: xcopa-eu
 
diff --git a/oellm/task_groups.py b/oellm/task_groups.py
index 8e5a846b..223376c6 100644
--- a/oellm/task_groups.py
+++ b/oellm/task_groups.py
@@ -49,6 +49,20 @@
     "ca": "cat_Latn",
     "eu": "eus_Latn",
     "gl": "glg_Latn",
+    # non-EU languages of xwinograd / xcopa / xstorycloze
+    "ar": "arb_Arab",
+    "hi": "hin_Deva",
+    "ht": "hat_Latn",
+    "id": "ind_Latn",
+    "jp": "jpn_Jpan",
+    "my": "mya_Mymr",
+    "qu": "que_Latn",
+    "sw": "swh_Latn",
+    "ta": "tam_Taml",
+    "te": "tel_Telu",
+    "th": "tha_Thai",
+    "vi": "vie_Latn",
+    "zh": "zho_Hans",
     # openai/MMMLU locale codes (lang_REGION)
     "de_de": "deu_Latn",
     "es_la": "spa_Latn",
@@ -148,18 +162,23 @@ def _canonical_language(code: str | None) -> str | None:
     return None
 
 
+_FLORES_PAIR = re.compile(
+    r"^flores200[:_]([a-z]{3}_[A-Z][a-z]{3})-([a-z]{3}_[A-Z][a-z]{3})"
+)
+
+
 def _resolve_task_languages(name: str, subset: str | None) -> list[str]:
     """Return the canonical language code(s) a task belongs to, if any.
 
-    Translation pairs (``flores200:src-tgt`` and
+    Translation pairs (``flores200:src-tgt``, ``flores200_src-tgt_bpb`` and
     ``opensubtitles_multi40_src_to_tgt``) resolve to their non-English side;
     every other task resolves via its ``subset``. Tasks with no recognisable
     language (e.g. English-only standard benchmarks) return [].
     """
-    if name.startswith("flores200:"):
-        pair = name.split(":", 1)[1]
+    flores = _FLORES_PAIR.match(name)
+    if flores:
         langs = [
-            _canonical_language(part) for part in pair.split("-") if part != "eng_Latn"
+            _canonical_language(part) for part in flores.groups() if part != "eng_Latn"
         ]
         return [lang for lang in langs if lang]
     if name.startswith("opensubtitles_multi40_"):
diff --git a/tests/test_bpb_multilingual.py b/tests/test_bpb_multilingual.py
new file mode 100644
index 00000000..5bac808e
--- /dev/null
+++ b/tests/test_bpb_multilingual.py
@@ -0,0 +1,180 @@
+"""Tests for the multilingual BPB tasks (the ``oellm-multilingual-bpb`` super group).
+
+The registry checks guard against drift between task-groups.yaml and the task
+YAMLs in custom_lm_eval_tasks/bpb/; the scoring checks pin the byte convention
+documented in bpb/utils.py on synthetic documents (no datasets or models).
+"""
+
+import importlib.util
+import math
+import re
+from importlib.resources import files
+
+import pytest
+import yaml
+
+from oellm.task_groups import _expand_task_groups
+
+BPB_DIR = files("oellm.resources") / "custom_lm_eval_tasks" / "bpb"
+
+
+def _load_utils():
+    spec = importlib.util.spec_from_file_location("bpb_utils", str(BPB_DIR / "utils.py"))
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+utils = _load_utils()
+
+
+def _defined_bpb_tasks() -> set[str]:
+    names = set()
+    for path in BPB_DIR.rglob("*.yaml"):
+        match = re.search(r"^task: (\S+)$", path.read_text(encoding="utf-8"), re.M)
+        if match:
+            names.add(match.group(1))
+    return names
+
+
+# --- registry -----------------------------------------------------------------
+
+
+def test_super_group_expands_to_every_benchmark():
+    jobs = _expand_task_groups(["oellm-multilingual-bpb"])
+    assert len(jobs) == 169
+    assert {j.suite for j in jobs} == {"lm-eval-harness"}
+    assert {j.n_shot for j in jobs} == {0}
+
+
+def test_every_scheduled_task_is_defined_and_reports_bpb():
+    task_metrics = yaml.safe_load(
+        (files("oellm.resources") / "task-groups.yaml").read_text()
+    )["task_metrics"]
+    defined = _defined_bpb_tasks()
+    for job in _expand_task_groups(["oellm-multilingual-bpb"]):
+        assert job.task in defined, f"{job.task} has no YAML in custom_lm_eval_tasks/bpb"
+        assert task_metrics.get(job.task) == "bits_per_byte", job.task
+
+
+def test_super_group_bracket_mirrors_oellm_multilingual():
+    """Scoping to one language selects that language's BPB counterpart of each
+    oellm-multilingual benchmark (the non-BPB test expects the same set)."""
+    tasks = {j.task for j in _expand_task_groups(["oellm-multilingual-bpb[deu_Latn]"])}
+    assert tasks == {
+        "belebele_deu_Latn_bpb",
+        "flores200_deu_Latn-eng_Latn_bpb",
+        "flores200_eng_Latn-deu_Latn_bpb",
+        "global_mmlu_de_bpb",
+        "include_base_44_german_bpb",
+        "mgsm_direct_de_bpb",
+    }
+
+
+def test_template_functions_exist_in_utils():
+    for path in BPB_DIR.glob("_*_template_yaml"):
+        for fn in re.findall(r"!function utils\.(\w+)", path.read_text(encoding="utf-8")):
+            assert callable(getattr(utils, fn, None)), f"{path.name}: utils.{fn}"
+
+
+def test_task_yamls_in_subdirectories_use_no_functions():
+    """`!function utils.x` resolves next to the YAML that declares it, so only
+    the templates beside bpb/utils.py may use it."""
+    for path in BPB_DIR.glob("*/*.yaml"):
+        assert "!function" not in path.read_text(encoding="utf-8"), path
+
+
+# --- scoring ------------------------------------------------------------------
+
+
+def _mc(*lls):
+    return [(ll, False) for ll in lls]
+
+
+def test_belebele_scores_gold_answer_text():
+    doc = {
+        "mc_answer1": "a",
+        "mc_answer2": "é",
+        "mc_answer3": "猫",
+        "mc_answer4": "d",
+        "correct_answer_num": "3",
+    }
+    out = utils.process_results_belebele(doc, _mc(-4, -3, -1, -5))
+    assert out == {"acc": 1.0, "acc_norm": 1.0, "bits_per_byte": (-1.0, 3)}
+
+
+def test_global_mmlu_letter_answer():
+    doc = {"option_a": "x", "option_b": "réponse", "option_c": "y", "option_d": "z"}
+    out = utils.process_results_global_mmlu({**doc, "answer": "B"}, _mc(-4, -1, -2, -3))
+    assert out["bits_per_byte"] == (-1.0, len("réponse".encode()))
+    malformed = utils.process_results_global_mmlu(
+        {**doc, "answer": ""}, _mc(-4, -1, -2, -3)
+    )
+    assert malformed["bits_per_byte"] == (0.0, 1)
+
+
+def test_include_integer_answer():
+    doc = {"option_a": "x", "option_b": "y", "option_c": "Antwort", "option_d": "z"}
+    out = utils.process_results_include({**doc, "answer": 2}, _mc(-4, -3, -1, -5))
+    assert out["bits_per_byte"] == (-1.0, 7)
+
+
+def test_xcopa_lowercases_choice_like_upstream():
+    doc = {"choice1": "See oli õrn.", "choice2": "Ärge.", "label": 1}
+    assert utils.doc_to_choice_xcopa(doc) == ["see oli õrn.", "ärge."]
+    out = utils.process_results_xcopa(doc, _mc(-2, -1))
+    assert out["bits_per_byte"] == (-1.0, len("ärge.".encode()))
+
+
+def test_xstorycloze_one_indexed_ending():
+    doc = {"sentence_quiz1": "a", "sentence_quiz2": "猫", "answer_right_ending": 2}
+    assert utils.process_results_xstorycloze(doc, _mc(-2, -1))["bits_per_byte"] == (
+        -1.0,
+        3,
+    )
+
+
+def test_xwinograd_counts_shared_continuation():
+    doc = {
+        "sentence": "_ était ici",
+        "option1": "Elle",
+        "option2": "La personne",
+        "answer": "1",
+    }
+    out = utils.process_results_winogrande(doc, _mc(-1, -2))
+    assert out["bits_per_byte"] == (-1.0, len("était ici".encode()))
+
+
+def test_mgsm_excludes_the_space_it_prepends():
+    doc = {"answer_number": 18.0}
+    assert utils.doc_to_target_mgsm(doc) == " 18"
+    assert utils.process_results_mgsm(doc, [(-2.0, False)]) == {
+        "bits_per_byte": (-2.0, 2)
+    }
+
+
+def test_flores_scores_reference_in_both_directions():
+    doc = {"sentence_deu_Latn": "Grüße", "sentence_eng_Latn": "Greetings"}
+    assert utils.doc_to_target_flores_xx_eng(doc) == " Greetings"
+    assert utils.process_results_flores_xx_eng(doc, [(-3.0, False)]) == {
+        "bits_per_byte": (-3.0, 9)
+    }
+    assert utils.doc_to_target_flores_eng_xx(doc) == " Grüße"
+    assert utils.process_results_flores_eng_xx(doc, [(-3.0, False)]) == {
+        "bits_per_byte": (-3.0, len("Grüße".encode()))
+    }
+
+
+def test_pairs_aggregate_to_corpus_level_bpb():
+    """lm-eval's bits_per_byte aggregation is byte-weighted: -sum(ll)/sum(bytes)/ln 2."""
+    pairs = [
+        utils.process_results_flores_xx_eng(
+            {"sentence_deu_Latn": "x", "sentence_eng_Latn": "a"}, [(-math.log(2), False)]
+        )["bits_per_byte"],
+        utils.process_results_flores_xx_eng(
+            {"sentence_deu_Latn": "x", "sentence_eng_Latn": "猫"},
+            [(-6 * math.log(2), False)],
+        )["bits_per_byte"],
+    ]
+    lls, n_bytes = zip(*pairs, strict=True)
+    assert -sum(lls) / sum(n_bytes) / math.log(2) == pytest.approx(1.75)
diff --git a/tests/test_language_groups.py b/tests/test_language_groups.py
index 16a81cee..8ab2c457 100644
--- a/tests/test_language_groups.py
+++ b/tests/test_language_groups.py
@@ -30,6 +30,7 @@
     "xcsqa-eu",
     "pawsx-eu",
     "xnli-eu",
+    "include-bpb",
     "mmmlu-eu",
 ]