[DRAFT][FOR REVIEW] Merging elliot eval platform with oellm

.github/sky/slurm-integration.yaml

@@ -127,7 +127,7 @@ run: |
       lm-eval torch transformers accelerate "datasets<4.0.0"
     UV_TOOL_DIR="$EVAL_BASE_DIR/.uv-tools" UV_TOOL_BIN_DIR="$EVAL_BASE_DIR/.venv/bin" \
       uv tool install --python 3.12 \
-        --with "langcodes[data]" --with "pillow" \
+        --with "langcodes[data]" --with "pillow" --with "xxhash<4" \
         "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git"
   fi
 

.github/workflows/ci.yml

@@ -63,7 +63,7 @@ jobs:
         run: uv python install 3.12
 
       - name: Install dependencies
-        run: uv sync --extra dev --extra eval-base
+        run: uv sync --extra dev
 
       - name: Run tests
-        run: uv run pytest tests/ -v --ignore=tests/integration/
+        run: uv run pytest tests/ -v --ignore=tests/integration/ --ignore=tests/test_datasets.py

.gitignore

@@ -16,3 +16,9 @@
 **/task_map_cache.json
 *.sif
 results/
+# SSH keys/certs must never be committed
+key_filename*
+*.pem
+id_rsa*
+id_ecdsa*
+id_ed25519*

containers/jupiter.def

@@ -13,17 +13,20 @@ From: nvcr.io/nvidia/pytorch:25.06-py3
     export UV_TOOL_BIN_DIR=/usr/local/bin
     uv --version
 
-    uv pip install --system --break-system-packages lm-eval \
+    uv pip install --system --break-system-packages lm-eval==0.4.12 \
+        latex2sympy2_extended \
         wandb sentencepiece tiktoken accelerate
 
     # lighteval as isolated tool (avoids dependency conflicts)
     export UV_TOOL_DIR=/opt/uv-tools
     # Install lighteval without [multilingual] extras first, then add multilingual deps
     # sudachipy (spacy[ja] dep) has no ARM64 wheel, so skip Japanese support
+    # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def)
     uv tool install --python 3.12 \
         --with "pillow" \
         --with "torch<2.9" \
-        "lighteval @ git+https://github.com/huggingface/lighteval.git"
+        --with "xxhash<4" \
+        "lighteval @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129"
     uv pip install --system --break-system-packages "spacy>=3.0.0,<4.0.0" jieba "underthesea>=6.0.0"
     uv pip install --system --break-system-packages nltk
 

containers/jureca.def

@@ -13,7 +13,8 @@ From: nvcr.io/nvidia/pytorch:25.06-py3
 
     # lm-eval and dependencies (system Python)
     uv pip install --system --break-system-packages \
-        lm-eval \
+        lm-eval==0.4.12 \
+        latex2sympy2_extended \
         transformers \
         "datasets<4.0.0" \
         wandb \
@@ -23,10 +24,12 @@ From: nvcr.io/nvidia/pytorch:25.06-py3
         nltk
 
     # lighteval as isolated tool (avoids dependency conflicts)
+    # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def)
     uv tool install --python 3.12 \
         --with "langcodes[data]" \
         --with "pillow" \
-        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git"
+        --with "xxhash<4" \
+        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129"
 
     # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time
     /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)"

containers/leonardo.def

@@ -13,7 +13,8 @@ From: nvcr.io/nvidia/pytorch:25.10-py3
 
     # lm-eval and dependencies (system Python)
     uv pip install --system --break-system-packages \
-        lm-eval \
+        lm-eval==0.4.12 \
+        latex2sympy2_extended \
         transformers \
         "datasets<4.0.0" \
         wandb \
@@ -23,10 +24,12 @@ From: nvcr.io/nvidia/pytorch:25.10-py3
         nltk
 
     # lighteval as isolated tool (avoids dependency conflicts)
+    # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def)
     uv tool install --python 3.12 \
         --with "langcodes[data]" \
         --with "pillow" \
-        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git"
+        --with "xxhash<4" \
+        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129"
 
     # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time
     /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)"

containers/lumi.def

@@ -13,7 +13,8 @@ From: rocm/pytorch:rocm6.4.1_ubuntu24.04_py3.12_pytorch_release_2.7.1
 
     # lm-eval and dependencies (system Python)
     uv pip install --system --break-system-packages \
-        lm-eval \
+        lm-eval==0.4.12 \
+        latex2sympy2_extended \
         transformers \
         "datasets<4.0.0" \
         wandb \
@@ -44,7 +45,7 @@ From: rocm/pytorch:rocm6.4.1_ubuntu24.04_py3.12_pytorch_release_2.7.1
         --with "langcodes[data]" \
         --with "pillow" \
         --with "xxhash<4" \
-        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git"
+        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129"
 
     # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time
     /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)"

containers/slurm-ci.def

@@ -13,7 +13,8 @@ From: nvcr.io/nvidia/pytorch:25.10-py3
 
     # lm-eval and dependencies (system Python)
     uv pip install --system --break-system-packages \
-        lm-eval \
+        lm-eval==0.4.12 \
+        latex2sympy2_extended \
         transformers \
         "datasets<4.0.0" \
         wandb \
@@ -23,10 +24,12 @@ From: nvcr.io/nvidia/pytorch:25.10-py3
         nltk
 
     # lighteval as isolated tool (avoids dependency conflicts)
+    # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def)
     uv tool install --python 3.12 \
         --with "langcodes[data]" \
         --with "pillow" \
-        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git"
+        --with "xxhash<4" \
+        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129"
 
     # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time
     /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)"

containers/snellius.def

@@ -13,7 +13,8 @@ From: nvcr.io/nvidia/pytorch:25.10-py3
 
     # lm-eval and dependencies (system Python)
     uv pip install --system --break-system-packages \
-        lm-eval \
+        lm-eval==0.4.12 \
+        latex2sympy2_extended \
         transformers \
         "datasets<4.0.0" \
         wandb \
@@ -23,10 +24,12 @@ From: nvcr.io/nvidia/pytorch:25.10-py3
         nltk
 
     # lighteval as isolated tool (avoids dependency conflicts)
+    # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def)
     uv tool install --python 3.12 \
         --with "langcodes[data]" \
         --with "pillow" \
-        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git"
+        --with "xxhash<4" \
+        "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129"
 
     # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time
     /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)"

docs/CONTAINERS.md

@@ -7,12 +7,47 @@ Apptainer containers are built automatically via GitHub Actions and stored on Hu
 ## How It Works
 
 1. Definition files live in `containers/<cluster>.def`
-2. On push to `main` (when `.def` files change), GitHub Actions provisions [Lambda Labs](https://lambdalabs.com/) GPU instances via [SkyPilot](https://skypilot.readthedocs.io/) and builds all containers in parallel
+2. Builds are triggered **manually** ("Run workflow" on the [build-and-push-apptainer](../.github/workflows/build-and-push-apptainer.yml) action — manual since PR #46). The workflow provisions [Lambda Labs](https://lambdalabs.com/) GPU instances via [SkyPilot](https://skypilot.readthedocs.io/) and builds all containers in parallel
 3. Built `.sif` images are uploaded to HuggingFace Hub
 4. Clusters pull the image specified in `oellm/resources/clusters.yaml` via `EVAL_CONTAINER_IMAGE`
 
 Images are compressed with zstd (level 3) via mksquashfs for a good balance of size and build speed.
 
+## What the Images Contain (and What They Don't)
+
+The shipped images provide exactly **two engines**: `lm-eval` (system Python)
+and `lighteval` (installed as an isolated uv tool to avoid the `datasets`
+version conflict). They do **not** contain `lmms-eval`, the `oellm` package
+(needed by contrib suites), or evalchemy — scheduling those suites in
+container mode is rejected by the environment pre-flight; pass `--venv-path`
+with a suitable venv instead (see [VENV.md](VENV.md)).
+
+## Image / Video / Audio Evaluation (lmms-eval)
+
+lmms-eval benchmarks require a custom venv:
+
+**Option 1 — Custom venv (recommended):**
+Follow the general-venv setup in [VENV.md](VENV.md) — note that `lmms-eval`
+is **not** provided by any pyproject extra; it must be installed editable
+from git (its wheel build drops required template files), alongside the
+`[text,image,audio]` extras.
+
+**Option 2 — Build a container with lmms-eval:**
+Extend a `.def` file — install lmms-eval **editable from git** (a plain
+`pip install lmms-eval` produces a broken install — the wheel drops
+`_default_template_yaml` files) and include the `oellm` package if contrib
+suites should run in the container:
+
+```singularity
+%post
+    uv pip install --system --break-system-packages \
+        lm-eval torch transformers accelerate "datasets<4.0.0"
+    uv pip install --system --break-system-packages \
+        -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git#egg=lmms-eval"
+```
+
+Then set `EVAL_CONTAINER_IMAGE` in `clusters.yaml` to point to this image.
+
 ## Adding a New Cluster
 
 1. Create `containers/<cluster>.def` with the appropriate base image: