From 1de79fb3ec8f2d3773b9f477e2f36ae3bc4db98a Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Thu, 5 Mar 2026 13:02:02 +0100 Subject: [PATCH 01/44] foundational refactoring and image modality mvp --- README.md | 88 +++++++++-- docs/CONTAINERS.md | 17 +++ docs/TASKS.md | 86 ++++++++++- docs/VENV.md | 24 ++- oellm/core/__init__.py | 5 + oellm/core/base_metric.py | 52 +++++++ oellm/core/base_model_adapter.py | 52 +++++++ oellm/core/base_task.py | 70 +++++++++ oellm/main.py | 16 +- oellm/resources/task-groups.yaml | 32 ++++ oellm/resources/template.sbatch | 29 ++++ pyproject.toml | 3 + requirements-venv.txt | 6 +- tests/test_base_interfaces.py | 235 +++++++++++++++++++++++++++++ tests/test_collect_results.py | 251 +++++++++++++++++++++++++++++++ tests/test_image_task_groups.py | 153 +++++++++++++++++++ 16 files changed, 1092 insertions(+), 27 deletions(-) create mode 100644 oellm/core/__init__.py create mode 100644 oellm/core/base_metric.py create mode 100644 oellm/core/base_model_adapter.py create mode 100644 oellm/core/base_task.py create mode 100644 tests/test_base_interfaces.py create mode 100644 tests/test_collect_results.py create mode 100644 tests/test_image_task_groups.py diff --git a/README.md b/README.md index cd3d6c56..5e2cda36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# OpenEuroLLM CLI (oellm) +# ELLIOT Evaluation Platform -A lightweight CLI for scheduling LLM evaluations across multiple HPC clusters using SLURM job arrays and Singularity containers. +A multimodal evaluation framework for scheduling LLM and VLM evaluations across HPC clusters. Extends the original oellm-cli with image modality support and a plugin interface for adding new benchmarks and modalities. ## Features @@ -8,7 +8,9 @@ A lightweight CLI for scheduling LLM evaluations across multiple HPC clusters us - **Collect results** and check for missing evaluations: `oellm collect-results` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA) -- **Automatic building and deployment of containers** +- **Image evaluation** via [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) +- **Plugin interface** (`BaseTask` / `BaseMetric` / `BaseModelAdapter`) for adding new benchmarks without touching core scheduling logic +- **Automatic building and deployment of containers** ## Quick Start @@ -44,17 +46,40 @@ In case you do not want to rely on the containers provided on a given cluster or Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/resources/task-groups.yaml). Each group specifies tasks, their n-shot settings, and HuggingFace dataset mappings. -Available task groups: -- `open-sci-0.01` - Standard benchmarks (COPA, MMLU, HellaSwag, ARC, etc.) -- `belebele-eu-5-shot` - Belebele European language tasks -- `flores-200-eu-to-eng` / `flores-200-eng-to-eu` - Translation tasks -- `global-mmlu-eu` - Global MMLU in EU languages -- `mgsm-eu` - Multilingual GSM benchmarks -- `generic-multilingual` - XWinograd, XCOPA, XStoryCloze -- `include` - INCLUDE benchmarks +### Text & Multilingual -Super groups combine multiple task groups: -- `oellm-multilingual` - All multilingual benchmarks combined +| Group | Description | Engine | +|---|---|---| +| `open-sci-0.01` | COPA, MMLU, HellaSwag, ARC, etc. | lm-eval | +| `belebele-eu-5-shot` | Belebele in 23 European languages | lm-eval | +| `flores-200-eu-to-eng` | EU → English translation | lighteval | +| `flores-200-eng-to-eu` | English → EU translation | lighteval | +| `global-mmlu-eu` | Global MMLU in EU languages | lm-eval | +| `mgsm-eu` | Multilingual GSM8K | lm-eval | +| `generic-multilingual` | XWinograd, XCOPA, XStoryCloze | lm-eval | +| `include` | INCLUDE benchmarks (44 languages) | lm-eval | + +Super groups: +- `oellm-multilingual` — all multilingual benchmarks combined + +### Image (lmms-eval) + +| Group | Benchmarks | Engine | +|---|---|---| +| `image-vqa` | VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista | lmms-eval | + +Image evaluation requires a container or venv with `lmms-eval` installed. Install the optional dependency: + +```bash +pip install "oellm[image]" +``` + +By default, the `llava_hf` model adapter is used (suitable for most HuggingFace VLMs). Override via `--slurm_template_var`: + +```bash +oellm schedule-eval --models "path/to/vlm" --task_groups "image-vqa" \ + --slurm_template_var '{"LMMS_MODEL_TYPE":"qwen_vl_chat"}' +``` ```bash # Use a task group @@ -65,6 +90,9 @@ oellm schedule-eval --models "model-name" --task_groups "belebele-eu-5-shot,glob # Use a super group oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" + +# Image evaluation +oellm schedule-eval --models "path/to/vlm" --task_groups "image-vqa" ``` ## SLURM Overrides @@ -158,13 +186,45 @@ git clone https://github.com/OpenEuroLLM/oellm-cli.git cd oellm-cli uv sync --extra dev -# Run dataset validation tests +# Run all unit tests +uv run pytest tests/ -v + +# Run dataset validation tests (requires network access) uv run pytest tests/test_datasets.py -v # Download-only mode for testing uv run oellm schedule-eval --models "EleutherAI/pythia-160m" --task_groups "open-sci-0.01" --download_only ``` +## Plugin Interface + +The `oellm.core` package provides abstract base classes for extending the platform without modifying core scheduling logic: + +```python +from oellm.core import BaseTask, BaseMetric, BaseModelAdapter +from oellm.task_groups import DatasetSpec + +# Register a new benchmark (one-liner if it's already in lmms-eval) +class MyTask(BaseTask): + @property + def name(self) -> str: + return "my_benchmark" + + @property + def suite(self) -> str: + return "lmms_eval" # or "lm_eval" / "lighteval" + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def dataset_specs(self) -> list[DatasetSpec]: + return [DatasetSpec(repo_id="org/my-dataset")] +``` + +See `oellm/core/` for full interface documentation. + ## Deploying containers Containers are deployed manually since [PR #46](https://github.com/OpenEuroLLM/oellm-cli/pull/46) to save costs. diff --git a/docs/CONTAINERS.md b/docs/CONTAINERS.md index b983f564..1fe61be3 100644 --- a/docs/CONTAINERS.md +++ b/docs/CONTAINERS.md @@ -13,6 +13,23 @@ Apptainer containers are built automatically via GitHub Actions and stored on Hu Images are compressed with zstd (level 3) via mksquashfs for a good balance of size and build speed. +## Image Evaluation (lmms-eval) + +Image benchmarks (`suite: lmms_eval`) require `lmms-eval` to be available in the execution environment. There are two ways to provide it: + +**Option 1 — Custom venv (recommended for development):** +Install `lmms-eval` via `requirements-venv.txt` and pass `--venv_path` to the CLI. See [VENV.md](VENV.md) for setup instructions. + +**Option 2 — Container with lmms-eval:** +Build a container `.def` file that includes `lmms-eval` alongside `lm-eval`: + +```singularity +%post + pip install lm-eval torch transformers accelerate "datasets<4.0.0" "lmms-eval>=0.2.4" +``` + +Then set `EVAL_CONTAINER_IMAGE` in `clusters.yaml` to point to this image. + ## Adding a New Cluster 1. Create `containers/.def` with the appropriate base image: diff --git a/docs/TASKS.md b/docs/TASKS.md index 6ed754cf..329db86c 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -4,15 +4,23 @@ Tasks are defined in `oellm/resources/task-groups.yaml`. Only tasks in this file are tested and guaranteed to work. The CLI parses this via `task_groups.py` and expands groups into `(task, n_shot, suite)` tuples for scheduling. +Three evaluation suites are supported: + +| Suite value | Engine | Use case | +|---|---|---| +| `lm_eval` | [lm-eval](https://github.com/EleutherAI/lm-evaluation-harness) | Text benchmarks | +| `lighteval` | [lighteval](https://github.com/huggingface/lighteval) | Translation / multilingual | +| `lmms_eval` | [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) | Image / VQA benchmarks | + ## YAML Structure ```yaml task_groups: my-group: description: "Short description" - suite: lm-eval-harness # or lighteval - n_shots: [5] # default for all tasks in group - dataset: org/dataset # default HF dataset for pre-download + suite: lm_eval # or lighteval, lmms_eval + n_shots: [5] # default for all tasks in group + dataset: org/dataset # default HF dataset for pre-download tasks: - task: task_name n_shots: [0, 5] # overrides group default @@ -20,7 +28,7 @@ task_groups: subset: subset_name # HF dataset config/subset ``` -## Adding a Task Group +## Adding a Text Task Group 1. Add your group to `oellm/resources/task-groups.yaml`: @@ -28,7 +36,7 @@ task_groups: task_groups: my-benchmark: description: "My custom benchmark" - suite: lm-eval-harness + suite: lm_eval n_shots: [0] dataset: huggingface/dataset-name tasks: @@ -44,12 +52,42 @@ task_groups: oellm schedule-eval --models "model-name" --task_groups "my-benchmark" ``` +## Adding an Image Task Group + +Image tasks use `suite: lmms_eval`. Each task must correspond to a task name recognized by lmms-eval. + +```yaml +task_groups: + my-image-benchmark: + description: "My image benchmark via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: vqav2_val_all + dataset: HuggingFaceM4/VQAv2 + - task: mmbench_en_dev + dataset: HuggingFaceM4/MMBench_00 +``` + +Run with: + +```bash +oellm schedule-eval --models "path/to/vlm" --task_groups "my-image-benchmark" +``` + +The model adapter defaults to `llava_hf`. Override via `--slurm_template_var`: + +```bash +oellm schedule-eval --models "path/to/vlm" --task_groups "my-image-benchmark" \ + --slurm_template_var '{"LMMS_MODEL_TYPE":"qwen_vl_chat"}' +``` + ## Field Reference | Field | Required | Level | Description | |-------|----------|-------|-------------| | `description` | Yes | group | Short description of the task group | -| `suite` | Yes | group | Evaluation suite: `lm-eval-harness` or `lighteval` | +| `suite` | Yes | group | Evaluation suite: `lm_eval`, `lighteval`, or `lmms_eval` | | `n_shots` | Yes | group or task | List of shot counts; must be set at group or task level | | `dataset` | Yes | group or task | HuggingFace dataset repo ID (required for pre-download and testing) | | `task` | Yes | task | Task name as recognized by the evaluation suite | @@ -62,3 +100,39 @@ oellm schedule-eval --models "model-name" --task_groups "my-benchmark" 2. **CI testing** - The test suite validates that all datasets in `task-groups.yaml` are accessible Tasks without a `dataset` field will not have their data pre-downloaded and are not covered by CI validation. + +## Plugin Interface (Advanced) + +For programmatic task registration without editing the YAML, use the `BaseTask` abstract base class from `oellm.core`: + +```python +from oellm.core import BaseTask +from oellm.task_groups import DatasetSpec + +class MyImageTask(BaseTask): + @property + def name(self) -> str: + return "my_benchmark" # canonical name used in CSV scheduling + + @property + def suite(self) -> str: + return "lmms_eval" # or "lm_eval" / "lighteval" + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def dataset_specs(self) -> list[DatasetSpec]: + return [DatasetSpec(repo_id="org/my-dataset")] +``` + +Override `engine_task_name` if the engine uses a different name than `name`: + +```python + @property + def engine_task_name(self) -> str: + return "my_benchmark_v2" # passed to --tasks; defaults to self.name +``` + +See `oellm/core/` for the full `BaseTask`, `BaseMetric`, and `BaseModelAdapter` interfaces. diff --git a/docs/VENV.md b/docs/VENV.md index bfdd39ca..c807bbcf 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -11,11 +11,13 @@ Instead of using pre-built containers, you can run evaluations with your own Pyt uv venv --python 3.12 /path/to/.venv ``` -2. Install lm-eval dependencies: +2. Install lm-eval and lmms-eval dependencies: ```bash uv pip install --python /path/to/.venv/bin/python -r requirements-venv.txt ``` + This installs `lm-eval`, `torch`, `transformers`, `accelerate`, `datasets<4.0.0`, and `lmms-eval`. + 3. Install lighteval as isolated tool (avoids datasets version conflict): ```bash UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ @@ -27,12 +29,26 @@ Instead of using pre-built containers, you can run evaluations with your own Pyt ## Usage ```bash +# Text evaluation oellm schedule-eval \ --models HuggingFaceTB/SmolLM2-135M-Instruct \ - --task_groups multilingual \ + --task_groups open-sci-0.01 \ + --venv_path /path/to/.venv + +# Image evaluation (lmms-eval) +oellm schedule-eval \ + --models path/to/vlm \ + --task_groups image-vqa \ --venv_path /path/to/.venv ``` -## Why Two Install Steps? +## Why Multiple Install Steps? + +lm-eval requires `datasets<4.0.0` while lighteval requires `datasets>=4.0.0`. Installing lighteval as an isolated uv tool (like the containers do) avoids this conflict. `lmms-eval` is compatible with `datasets<4.0.0` and can be installed alongside lm-eval in the same venv. + +## Dependency Summary -lm-eval requires `datasets<4.0.0` while lighteval requires `datasets>=4.0.0`. Installing lighteval as an isolated uv tool (like the containers do) avoids this conflict. +| Package | Install method | Reason | +|---|---|---| +| `lm-eval`, `torch`, `transformers`, `accelerate`, `datasets<4.0.0`, `lmms-eval` | `uv pip install -r requirements-venv.txt` | lm-eval + image eval, compatible dataset pin | +| `lighteval[multilingual]` | `uv tool install` (isolated) | Requires `datasets>=4.0.0` — must be isolated | diff --git a/oellm/core/__init__.py b/oellm/core/__init__.py new file mode 100644 index 00000000..08a238dd --- /dev/null +++ b/oellm/core/__init__.py @@ -0,0 +1,5 @@ +from oellm.core.base_metric import BaseMetric +from oellm.core.base_model_adapter import BaseModelAdapter +from oellm.core.base_task import BaseTask + +__all__ = ["BaseTask", "BaseMetric", "BaseModelAdapter"] diff --git a/oellm/core/base_metric.py b/oellm/core/base_metric.py new file mode 100644 index 00000000..3f63b618 --- /dev/null +++ b/oellm/core/base_metric.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod + + +class BaseMetric(ABC): + """Abstract base class for custom metric implementations. + + Implement this when an evaluation task requires a metric not natively + supported by lm-eval or lmms-eval (e.g. custom safety scores for T4.4, + or domain-specific metrics for T4.3). + + The ``compute`` method must return a scalar in [0, 1] by convention, + though higher-range metrics (e.g. OCRBench score /1000) are allowed when + the metric name makes the range unambiguous. + + Example:: + + class ExactMatchMetric(BaseMetric): + @property + def name(self) -> str: + return "exact_match" + + def compute( + self, + predictions: list[str], + references: list[str], + ) -> float: + if not predictions: + return 0.0 + correct = sum(p == r for p, r in zip(predictions, references)) + return correct / len(predictions) + """ + + @property + @abstractmethod + def name(self) -> str: + """Unique metric identifier, e.g. ``"vqa_score"`` or ``"anls"``.""" + + @abstractmethod + def compute( + self, + predictions: list[str], + references: list[str], + ) -> float: + """Compute the metric score. + + Args: + predictions: Model-generated answers (one per sample). + references: Ground-truth answers (one per sample). + + Returns: + Scalar score. Conventionally in [0, 1]; higher is better. + """ diff --git a/oellm/core/base_model_adapter.py b/oellm/core/base_model_adapter.py new file mode 100644 index 00000000..11f8a8cb --- /dev/null +++ b/oellm/core/base_model_adapter.py @@ -0,0 +1,52 @@ +from abc import ABC, abstractmethod +from pathlib import Path + + +class BaseModelAdapter(ABC): + """Abstract base class for model adapters. + + An adapter's sole responsibility is translating a model path/config into + the engine-specific argument strings passed on the command line. The + scheduling engine reads these strings and injects them into the sbatch + template — adapters never call the engines directly. + + This is the single integration point for adding new or proprietary models: + implement the two abstract methods and the adapter works with any engine + the platform supports. + + Example:: + + class HFModelAdapter(BaseModelAdapter): + def __init__(self, path: str, trust_remote_code: bool = True): + self._path = path + self._trust = trust_remote_code + + @property + def model_path(self) -> str: + return self._path + + def to_lm_eval_args(self) -> str: + return f"pretrained={self._path},trust_remote_code={self._trust}" + + def to_lmms_eval_args(self) -> str: + return f"pretrained={self._path}" + """ + + @property + @abstractmethod + def model_path(self) -> str | Path: + """Path to the model weights or HuggingFace repo ID.""" + + @abstractmethod + def to_lm_eval_args(self) -> str: + """Return the ``--model_args`` string for lm-eval-harness. + + Example: ``"pretrained=/path/to/model,trust_remote_code=True"`` + """ + + @abstractmethod + def to_lmms_eval_args(self) -> str: + """Return the ``--model_args`` string for lmms-eval. + + Example: ``"pretrained=/path/to/model"`` + """ diff --git a/oellm/core/base_task.py b/oellm/core/base_task.py new file mode 100644 index 00000000..df750f2d --- /dev/null +++ b/oellm/core/base_task.py @@ -0,0 +1,70 @@ +from abc import ABC, abstractmethod + +from oellm.task_groups import DatasetSpec + + +class BaseTask(ABC): + """Abstract base class for evaluation task plugins. + + Subclasses represent a single logical evaluation task and provide + the metadata needed for scheduling and dataset pre-download. + + This class is forward-looking: it is not yet consumed by the scheduling + engine. Teams building T4.2–T4.5 integrations should subclass BaseTask + to register new tasks without touching the YAML or core scheduling logic. + + Example (benchmark already in lmms-eval):: + + class VQAv2Task(BaseTask): + @property + def name(self) -> str: + return "vqav2_val_all" + + @property + def suite(self) -> str: + return "lmms_eval" + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def dataset_specs(self) -> list[DatasetSpec]: + return [DatasetSpec(repo_id="HuggingFaceM4/VQAv2")] + """ + + @property + @abstractmethod + def name(self) -> str: + """Canonical task name, used as the CSV task_path column value.""" + + @property + @abstractmethod + def suite(self) -> str: + """Evaluation suite identifier. + + Must be one of: ``lm_eval``, ``lighteval``, ``lmms_eval``. + """ + + @property + @abstractmethod + def n_shots(self) -> list[int]: + """List of n-shot values to evaluate at.""" + + @property + def engine_task_name(self) -> str: + """Task name as passed to the eval engine CLI. + + Defaults to ``name``. Override when the engine's registered task name + differs from the canonical task name used in the CSV / YAML. + """ + return self.name + + @property + def dataset_specs(self) -> list[DatasetSpec]: + """Dataset specifications for pre-download. + + Return an empty list if no pre-download is required (e.g. the dataset + is already available on the compute nodes or pre-downloaded elsewhere). + """ + return [] diff --git a/oellm/main.py b/oellm/main.py index e23b3ee2..915c029b 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -429,6 +429,16 @@ def _resolve_metric( ) -> tuple[float | None, str | None]: """Return (value, metric_name) for task_name from result_dict.""" + # Normalise lmms-eval task-scoped metric keys so lm-eval and lmms-eval + # output is handled identically. lmms-eval writes keys like + # "vqav2/vqa_score,none"; strip the "task_name/" prefix so the lookup + # below sees "vqa_score,none" regardless of engine. Keys without "/" + # (lm-eval format) are passed through unchanged. + result_dict = { + (k.split("/", 1)[1] if "/" in k else k): v + for k, v in result_dict.items() + } + # Skip non-metric keys; lm-eval uses suffixes like ",none" or ",remove_whitespace" def _first_numeric(d: dict, *candidates: str) -> tuple[float | None, str | None]: for c in candidates: @@ -500,8 +510,10 @@ def _first_matching_prefix( with open(json_file) as f: data = json.load(f) - # Extract model name/path - model_name = data.get("model_name", "unknown") + # Extract model name/path. + # lmms-eval sets model_name to the adapter type (e.g. "llava_hf"), + # not the checkpoint path; the actual path is in model_name_or_path. + model_name = data.get("model_name_or_path") or data.get("model_name", "unknown") # Extract results for each task results = data.get("results", {}) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 1742e450..1757df9e 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -10,6 +10,15 @@ task_metrics: commonsense_qa: acc hellaswag: acc_norm piqa: acc_norm + # lmms-eval image benchmark metrics + vqav2_val_all: vqa_score + mmbench_en_dev: acc + mmmu_val: acc + chartqa: relaxed_accuracy + docvqa_val: anls + textvqa_val: acc + ocrbench: score + mathvista_testmini: acc task_groups: open-sci-0.01: @@ -293,6 +302,29 @@ task_groups: - task: include_base_44_ukrainian subset: Ukrainian + # ── Image Modality (lmms-eval) ──────────────────────────────────────────── + image-vqa: + description: "Image VQA benchmarks via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: vqav2_val_all + dataset: HuggingFaceM4/VQAv2 + - task: mmbench_en_dev + dataset: HuggingFaceM4/MMBench_00 + - task: mmmu_val + dataset: MMMU/MMMU + - task: chartqa + dataset: HuggingFaceM4/ChartQA + - task: docvqa_val + dataset: eliolio/docvqa + - task: textvqa_val + dataset: facebook/textvqa + - task: ocrbench + dataset: echo840/OCRBench + - task: mathvista_testmini + dataset: AI4Math/MathVista + super_groups: oellm-multilingual: description: "Combined Belebele EU set plus multilingual benchmarks" diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index ffd676c3..98b16592 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -157,6 +157,35 @@ do ${{LIMIT:+--max-samples $LIMIT}} fi ;; + lmms_eval|lmms-eval) + # lmms-eval is used for image/video/audio benchmarks. + # Model type defaults to llava_hf (HuggingFace LLaVA-style VLMs). + # Override via LMMS_MODEL_TYPE in clusters.yaml or --slurm_template_var. + LMMS_MODEL_TYPE="${{LMMS_MODEL_TYPE:-llava_hf}}" + OUTPUT_JSON="{evals_dir}/$(openssl rand -hex 5).json" + + if [ -n "$VENV_PATH" ]; then + source "$VENV_PATH/bin/activate" + python -m lmms_eval \ + --model "$LMMS_MODEL_TYPE" \ + --model_args "pretrained=$model_path" \ + --tasks "$task_path" \ + --num_fewshot "$n_shot" \ + --output_path "$OUTPUT_JSON" \ + ${{LIMIT:+--limit $LIMIT}} + else + singularity exec $SINGULARITY_ARGS \ + --bind $BIND_PATHS \ + $EVAL_SIF_PATH \ + python -m lmms_eval \ + --model "$LMMS_MODEL_TYPE" \ + --model_args "pretrained=$model_path" \ + --tasks "$task_path" \ + --num_fewshot "$n_shot" \ + --output_path "$OUTPUT_JSON" \ + ${{LIMIT:+--limit $LIMIT}} + fi + ;; *) echo "[warning] Unknown evaluation suite '$eval_suite'. Skipping." ;; diff --git a/pyproject.toml b/pyproject.toml index 9225a20e..36e10d98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,9 @@ dev = [ "pytest-timeout>=2.3.1", "pre-commit", ] +image = [ + "lmms-eval>=0.2.4", +] [project.scripts] oellm = "oellm.main:main" diff --git a/requirements-venv.txt b/requirements-venv.txt index e273bd96..c55e4ddd 100644 --- a/requirements-venv.txt +++ b/requirements-venv.txt @@ -1,4 +1,4 @@ -# Dependencies for lm-eval (install in venv) +# Dependencies for lm-eval and lmms-eval (install in venv) # Install with: uv pip install -r requirements-venv.txt lm-eval torch @@ -6,6 +6,10 @@ transformers accelerate datasets<4.0.0 +# lmms-eval: image/video/audio evaluation engine (required for image-vqa task group) +# lmms-eval is compatible with datasets<4.0.0; install alongside lm-eval. +lmms-eval>=0.2.4 + # lighteval must be installed separately as a uv tool to avoid datasets version conflict: # UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ # uv tool install --python 3.12 --with "langcodes[data]" --with "pillow" \ diff --git a/tests/test_base_interfaces.py b/tests/test_base_interfaces.py new file mode 100644 index 00000000..84f8648d --- /dev/null +++ b/tests/test_base_interfaces.py @@ -0,0 +1,235 @@ +import pytest + +from oellm.core import BaseMetric, BaseModelAdapter, BaseTask +from oellm.task_groups import DatasetSpec + + +# ── Concrete implementations for testing ───────────────────────────────────── + + +class MinimalTask(BaseTask): + @property + def name(self) -> str: + return "my_task" + + @property + def suite(self) -> str: + return "lmms_eval" + + @property + def n_shots(self) -> list[int]: + return [0] + + +class TaskWithDataset(MinimalTask): + @property + def dataset_specs(self) -> list[DatasetSpec]: + return [DatasetSpec(repo_id="org/repo", subset="val")] + + +class TaskWithCustomEngineName(MinimalTask): + @property + def engine_task_name(self) -> str: + return "engine_specific_name" + + +class ExactMatchMetric(BaseMetric): + @property + def name(self) -> str: + return "exact_match" + + def compute(self, predictions: list[str], references: list[str]) -> float: + if not predictions: + return 0.0 + return sum(p == r for p, r in zip(predictions, references)) / len(predictions) + + +class HFAdapter(BaseModelAdapter): + @property + def model_path(self) -> str: + return "/models/llava" + + def to_lm_eval_args(self) -> str: + return f"pretrained={self.model_path},trust_remote_code=True" + + def to_lmms_eval_args(self) -> str: + return f"pretrained={self.model_path}" + + +# ── BaseTask tests ──────────────────────────────────────────────────────────── + + +class TestBaseTask: + def test_abstract_class_cannot_be_instantiated(self): + with pytest.raises(TypeError): + BaseTask() # type: ignore[abstract] + + def test_concrete_subclass_instantiates(self): + t = MinimalTask() + assert t.name == "my_task" + assert t.suite == "lmms_eval" + assert t.n_shots == [0] + + def test_engine_task_name_defaults_to_name(self): + t = MinimalTask() + assert t.engine_task_name == t.name + + def test_engine_task_name_can_be_overridden(self): + t = TaskWithCustomEngineName() + assert t.name == "my_task" + assert t.engine_task_name == "engine_specific_name" + + def test_dataset_specs_defaults_to_empty_list(self): + t = MinimalTask() + assert t.dataset_specs == [] + + def test_dataset_specs_can_be_overridden(self): + t = TaskWithDataset() + assert len(t.dataset_specs) == 1 + spec = t.dataset_specs[0] + assert spec.repo_id == "org/repo" + assert spec.subset == "val" + + def test_missing_abstract_name_raises(self): + class BadTask(BaseTask): + @property + def suite(self) -> str: + return "lm_eval" + + @property + def n_shots(self) -> list[int]: + return [5] + + with pytest.raises(TypeError): + BadTask() # type: ignore[abstract] + + def test_missing_abstract_suite_raises(self): + class BadTask(BaseTask): + @property + def name(self) -> str: + return "task" + + @property + def n_shots(self) -> list[int]: + return [5] + + with pytest.raises(TypeError): + BadTask() # type: ignore[abstract] + + def test_missing_abstract_n_shots_raises(self): + class BadTask(BaseTask): + @property + def name(self) -> str: + return "task" + + @property + def suite(self) -> str: + return "lm_eval" + + with pytest.raises(TypeError): + BadTask() # type: ignore[abstract] + + +# ── BaseMetric tests ────────────────────────────────────────────────────────── + + +class TestBaseMetric: + def test_abstract_class_cannot_be_instantiated(self): + with pytest.raises(TypeError): + BaseMetric() # type: ignore[abstract] + + def test_concrete_subclass_instantiates(self): + m = ExactMatchMetric() + assert m.name == "exact_match" + + def test_compute_perfect_score(self): + m = ExactMatchMetric() + assert m.compute(["a", "b", "c"], ["a", "b", "c"]) == 1.0 + + def test_compute_zero_score(self): + m = ExactMatchMetric() + assert m.compute(["a", "b"], ["x", "y"]) == 0.0 + + def test_compute_partial_score(self): + m = ExactMatchMetric() + assert m.compute(["a", "b"], ["a", "x"]) == pytest.approx(0.5) + + def test_compute_empty_returns_zero(self): + m = ExactMatchMetric() + assert m.compute([], []) == 0.0 + + def test_missing_abstract_name_raises(self): + class BadMetric(BaseMetric): + def compute(self, predictions, references): + return 0.0 + + with pytest.raises(TypeError): + BadMetric() # type: ignore[abstract] + + def test_missing_abstract_compute_raises(self): + class BadMetric(BaseMetric): + @property + def name(self): + return "bad" + + with pytest.raises(TypeError): + BadMetric() # type: ignore[abstract] + + +# ── BaseModelAdapter tests ──────────────────────────────────────────────────── + + +class TestBaseModelAdapter: + def test_abstract_class_cannot_be_instantiated(self): + with pytest.raises(TypeError): + BaseModelAdapter() # type: ignore[abstract] + + def test_concrete_subclass_instantiates(self): + a = HFAdapter() + assert a.model_path == "/models/llava" + + def test_to_lm_eval_args_contains_path(self): + a = HFAdapter() + args = a.to_lm_eval_args() + assert "pretrained=/models/llava" in args + assert "trust_remote_code=True" in args + + def test_to_lmms_eval_args_contains_path(self): + a = HFAdapter() + args = a.to_lmms_eval_args() + assert args == "pretrained=/models/llava" + + def test_missing_abstract_model_path_raises(self): + class BadAdapter(BaseModelAdapter): + def to_lm_eval_args(self): + return "" + + def to_lmms_eval_args(self): + return "" + + with pytest.raises(TypeError): + BadAdapter() # type: ignore[abstract] + + def test_missing_abstract_to_lm_eval_args_raises(self): + class BadAdapter(BaseModelAdapter): + @property + def model_path(self): + return "/path" + + def to_lmms_eval_args(self): + return "" + + with pytest.raises(TypeError): + BadAdapter() # type: ignore[abstract] + + def test_missing_abstract_to_lmms_eval_args_raises(self): + class BadAdapter(BaseModelAdapter): + @property + def model_path(self): + return "/path" + + def to_lm_eval_args(self): + return "" + + with pytest.raises(TypeError): + BadAdapter() # type: ignore[abstract] diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py new file mode 100644 index 00000000..13f78552 --- /dev/null +++ b/tests/test_collect_results.py @@ -0,0 +1,251 @@ +"""Tests for collect_results() covering both lm-eval and lmms-eval output formats.""" + +import json +from pathlib import Path + +import pandas as pd +import pytest + +from oellm.main import collect_results + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def write_result(results_dir: Path, data: dict, filename: str = "result.json") -> None: + (results_dir / filename).write_text(json.dumps(data)) + + +def run_collect(tmp_path: Path, *json_payloads: dict) -> pd.DataFrame: + """Write JSON result files and run collect_results; return CSV as DataFrame.""" + results_dir = tmp_path / "results" + results_dir.mkdir() + for i, data in enumerate(json_payloads): + write_result(results_dir, data, filename=f"result_{i}.json") + + output_csv = str(tmp_path / "out.csv") + collect_results(str(results_dir), output_csv=output_csv) + + csv_path = Path(output_csv) + if not csv_path.exists(): + return pd.DataFrame() + return pd.read_csv(csv_path) + + +# ── lm-eval format (baseline — must remain unchanged) ──────────────────────── + + +class TestCollectResultsLmEvalFormat: + def test_standard_acc_metric(self, tmp_path): + data = { + "model_name": "/path/to/model", + "results": {"mmlu": {"acc,none": 0.75}}, + "n-shot": {"mmlu": 5}, + } + df = run_collect(tmp_path, data) + assert len(df) == 1 + row = df.iloc[0] + assert row["model_name"] == "/path/to/model" + assert row["task"] == "mmlu" + assert row["performance"] == pytest.approx(0.75) + assert row["n_shot"] == 5 + + def test_acc_norm_metric(self, tmp_path): + data = { + "model_name": "/path/to/model", + "results": {"arc_challenge": {"acc_norm,none": 0.60}}, + "n-shot": {"arc_challenge": 10}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.60) + + def test_multiple_tasks_in_one_file(self, tmp_path): + data = { + "model_name": "/models/llm", + "results": { + "hellaswag": {"acc_norm,none": 0.80}, + "arc_easy": {"acc_norm,none": 0.85}, + }, + "n-shot": {"hellaswag": 10, "arc_easy": 10}, + } + df = run_collect(tmp_path, data) + assert len(df) == 2 + assert set(df["task"].tolist()) == {"hellaswag", "arc_easy"} + + def test_multiple_json_files_aggregated(self, tmp_path): + data1 = { + "model_name": "/model", + "results": {"mmlu": {"acc,none": 0.75}}, + "n-shot": {"mmlu": 5}, + } + data2 = { + "model_name": "/model", + "results": {"hellaswag": {"acc_norm,none": 0.80}}, + "n-shot": {"hellaswag": 10}, + } + df = run_collect(tmp_path, data1, data2) + assert len(df) == 2 + + def test_no_results_returns_empty(self, tmp_path): + results_dir = tmp_path / "results" + results_dir.mkdir() + output_csv = str(tmp_path / "out.csv") + collect_results(str(results_dir), output_csv=output_csv) + assert not Path(output_csv).exists() + + def test_model_name_fallback_when_no_path_field(self, tmp_path): + """lm-eval has no model_name_or_path; must fall back to model_name.""" + data = { + "model_name": "/path/to/model", + "results": {"hellaswag": {"acc_norm,none": 0.70}}, + "n-shot": {"hellaswag": 10}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["model_name"] == "/path/to/model" + + +# ── lmms-eval format (Phase 2 additions) ───────────────────────────────────── + + +class TestCollectResultsLmmsEvalFormat: + def test_model_name_or_path_takes_priority_over_model_name(self, tmp_path): + """lmms-eval sets model_name to adapter type; real path is in model_name_or_path.""" + data = { + "model_name": "llava_hf", + "model_name_or_path": "/checkpoints/llava-1.5-7b", + "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, + "n-shot": {"vqav2_val_all": 0}, + } + df = run_collect(tmp_path, data) + assert len(df) == 1 + assert df.iloc[0]["model_name"] == "/checkpoints/llava-1.5-7b" + + def test_task_scoped_vqa_score_key_resolved(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, + "n-shot": {"vqav2_val_all": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.82) + assert df.iloc[0]["task"] == "vqav2_val_all" + + def test_task_scoped_acc_key_resolved(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"mmbench_en_dev": {"mmbench_en_dev/acc,none": 0.75}}, + "n-shot": {"mmbench_en_dev": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.75) + assert df.iloc[0]["task"] == "mmbench_en_dev" + + def test_mmmu_task_scoped_key(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"mmmu_val": {"mmmu/acc,none": 0.55}}, + "n-shot": {"mmmu_val": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.55) + assert df.iloc[0]["task"] == "mmmu_val" + + def test_chartqa_relaxed_accuracy(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"chartqa": {"chartqa/relaxed_accuracy,none": 0.68}}, + "n-shot": {"chartqa": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.68) + + def test_docvqa_anls_metric(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"docvqa_val": {"docvqa_val/anls,none": 0.91}}, + "n-shot": {"docvqa_val": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.91) + + def test_ocrbench_score_metric(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"ocrbench": {"ocrbench/score,none": 512.0}}, + "n-shot": {"ocrbench": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(512.0) + + def test_mathvista_acc_metric(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"mathvista_testmini": {"mathvista_testmini/acc,none": 0.49}}, + "n-shot": {"mathvista_testmini": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.49) + + def test_multiple_image_tasks_in_one_file(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": { + "vqav2_val_all": {"vqav2/vqa_score,none": 0.82}, + "mmbench_en_dev": {"mmbench_en_dev/acc,none": 0.75}, + "chartqa": {"chartqa/relaxed_accuracy,none": 0.68}, + }, + "n-shot": { + "vqav2_val_all": 0, + "mmbench_en_dev": 0, + "chartqa": 0, + }, + } + df = run_collect(tmp_path, data) + assert len(df) == 3 + assert set(df["task"].tolist()) == {"vqav2_val_all", "mmbench_en_dev", "chartqa"} + + def test_lmeval_and_lmms_eval_results_aggregated(self, tmp_path): + """Both lm-eval and lmms-eval JSON files can coexist in the same results dir.""" + lm_eval_data = { + "model_name": "/path/to/model", + "results": {"mmlu": {"acc,none": 0.75}}, + "n-shot": {"mmlu": 5}, + } + lmms_eval_data = { + "model_name": "llava_hf", + "model_name_or_path": "/path/to/model", + "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, + "n-shot": {"vqav2_val_all": 0}, + } + df = run_collect(tmp_path, lm_eval_data, lmms_eval_data) + assert len(df) == 2 + assert set(df["task"].tolist()) == {"mmlu", "vqav2_val_all"} + + def test_empty_model_name_or_path_falls_back_to_model_name(self, tmp_path): + """Empty string for model_name_or_path should fall back to model_name.""" + data = { + "model_name": "llava_hf", + "model_name_or_path": "", + "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.80}}, + "n-shot": {"vqav2_val_all": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["model_name"] == "llava_hf" + + def test_n_shot_zero_preserved(self, tmp_path): + data = { + "model_name": "llava_hf", + "model_name_or_path": "/models/llava", + "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, + "n-shot": {"vqav2_val_all": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["n_shot"] == 0 diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py new file mode 100644 index 00000000..a44b01d4 --- /dev/null +++ b/tests/test_image_task_groups.py @@ -0,0 +1,153 @@ +import os +import sys +from importlib.resources import files +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from oellm.task_groups import ( + _collect_dataset_specs, + _expand_task_groups, + get_all_task_group_names, +) + +IMAGE_TASK_GROUP = "image-vqa" + +EXPECTED_TASKS = { + "vqav2_val_all", + "mmbench_en_dev", + "mmmu_val", + "chartqa", + "docvqa_val", + "textvqa_val", + "ocrbench", + "mathvista_testmini", +} + +EXPECTED_DATASETS = { + "HuggingFaceM4/VQAv2", + "HuggingFaceM4/MMBench_00", + "MMMU/MMMU", + "HuggingFaceM4/ChartQA", + "eliolio/docvqa", + "facebook/textvqa", + "echo840/OCRBench", + "AI4Math/MathVista", +} + + +class TestImageTaskGroupInRegistry: + def test_image_vqa_present_in_yaml(self): + all_groups = get_all_task_group_names() + assert IMAGE_TASK_GROUP in all_groups + + def test_image_vqa_suite_is_lmms_eval(self): + data = yaml.safe_load( + (files("oellm.resources") / "task-groups.yaml").read_text() + ) + suite = data["task_groups"][IMAGE_TASK_GROUP]["suite"] + assert suite == "lmms_eval" + + def test_image_vqa_has_eight_tasks(self): + data = yaml.safe_load( + (files("oellm.resources") / "task-groups.yaml").read_text() + ) + tasks = data["task_groups"][IMAGE_TASK_GROUP]["tasks"] + assert len(tasks) == 8 + + +class TestImageTaskGroupExpansion: + def test_expands_to_correct_task_names(self): + results = _expand_task_groups([IMAGE_TASK_GROUP]) + task_names = {r.task for r in results} + assert task_names == EXPECTED_TASKS + + def test_all_tasks_have_zero_shot(self): + results = _expand_task_groups([IMAGE_TASK_GROUP]) + for r in results: + assert r.n_shot == 0, f"{r.task} has n_shot={r.n_shot}, expected 0" + + def test_all_tasks_route_to_lmms_eval(self): + results = _expand_task_groups([IMAGE_TASK_GROUP]) + for r in results: + assert r.suite == "lmms_eval", ( + f"{r.task} has suite='{r.suite}', expected 'lmms_eval'" + ) + + def test_expand_unknown_group_raises(self): + with pytest.raises(ValueError, match="Unknown task group"): + _expand_task_groups(["nonexistent-group"]) + + +class TestImageTaskGroupDatasetSpecs: + def test_all_expected_datasets_present(self): + specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert repo_ids == EXPECTED_DATASETS + + def test_no_duplicate_dataset_specs(self): + specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) + keys = [(s.repo_id, s.subset) for s in specs] + assert len(keys) == len(set(keys)), "Duplicate dataset specs found" + + def test_vqav2_dataset_included(self): + specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert "HuggingFaceM4/VQAv2" in repo_ids + + def test_mmmu_dataset_included(self): + specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert "MMMU/MMMU" in repo_ids + + +class TestImageTaskGroupScheduleEvals: + """Verify image-vqa integrates with the schedule_evals dry-run path.""" + + def test_schedule_evals_dry_run_image_vqa(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.main._load_cluster_env"), + patch("oellm.main._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + task_groups=IMAGE_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + sbatch_content = sbatch_files[0].read_text() + # The generated sbatch should contain the lmms_eval suite value + assert "lmms_eval" in sbatch_content + + def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): + import pandas as pd + + from oellm.main import schedule_evals + + with ( + patch("oellm.main._load_cluster_env"), + patch("oellm.main._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + task_groups=IMAGE_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + csv_files = list(tmp_path.glob("**/jobs.csv")) + assert len(csv_files) == 1 + df = pd.read_csv(csv_files[0]) + assert set(df["eval_suite"].unique()) == {"lmms_eval"} + assert set(df["task_path"].unique()) == EXPECTED_TASKS From ae6129a1cac75b130ee7421bc47cadee0863e468 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Mon, 9 Mar 2026 12:01:24 +0100 Subject: [PATCH 02/44] updated readme --- README.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5e2cda36..43785e37 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/reso Super groups: - `oellm-multilingual` — all multilingual benchmarks combined -### Image (lmms-eval) +### Image | Group | Benchmarks | Engine | |---|---|---| @@ -74,13 +74,6 @@ Image evaluation requires a container or venv with `lmms-eval` installed. Instal pip install "oellm[image]" ``` -By default, the `llava_hf` model adapter is used (suitable for most HuggingFace VLMs). Override via `--slurm_template_var`: - -```bash -oellm schedule-eval --models "path/to/vlm" --task_groups "image-vqa" \ - --slurm_template_var '{"LMMS_MODEL_TYPE":"qwen_vl_chat"}' -``` - ```bash # Use a task group oellm schedule-eval --models "model-name" --task_groups "open-sci-0.01" @@ -92,7 +85,7 @@ oellm schedule-eval --models "model-name" --task_groups "belebele-eu-5-shot,glob oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" # Image evaluation -oellm schedule-eval --models "path/to/vlm" --task_groups "image-vqa" +oellm schedule-eval --models "model-name" --task_groups "image-vqa" ``` ## SLURM Overrides From 84a8857e59b065d2aefe77afcfe5f2e4e3054711 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Mon, 9 Mar 2026 12:52:22 +0100 Subject: [PATCH 03/44] fix readme and added access tutorial to Leonardo cluster --- README.md | 20 +++++++++------- docs/LEONARDO.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 docs/LEONARDO.md diff --git a/README.md b/README.md index 43785e37..f59c2d9d 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across ```bash # Install the package -uv tool install -p 3.12 git+https://github.com/OpenEuroLLM/oellm-cli.git +uv tool install -p 3.12 git+https://github.com/elliot-project/elliot-cli.git # Run evaluations using a task group (recommended) oellm schedule-eval \ @@ -143,7 +143,7 @@ oellm schedule-eval --eval_csv_path custom_evals.csv ### General Installation ```bash -uv tool install -p 3.12 git+https://github.com/OpenEuroLLM/oellm-cli.git +uv tool install -p 3.12 git+https://github.com/elliot-project/elliot-cli.git ``` Update to latest: @@ -162,8 +162,12 @@ export UV_PYTHON_INSTALL_DIR="/p/project1//$USER/.local/share/uv/python export UV_TOOL_DIR="/p/project1//$USER/.cache/uv-tool-cache" ``` -## Supported Clusters: -We support: Leonardo, Lumi, and Jureca +## Supported Clusters + +We support: Leonardo, LUMI, and JURECA + +Cluster-specific access guides: +- [Leonardo HPC](docs/LEONARDO.md) ## CLI Options @@ -175,8 +179,8 @@ oellm schedule-eval --help ```bash # Clone and install in dev mode -git clone https://github.com/OpenEuroLLM/oellm-cli.git -cd oellm-cli +git clone https://github.com/elliot-project/elliot-cli.git +cd elliot-cli uv sync --extra dev # Run all unit tests @@ -220,9 +224,9 @@ See `oellm/core/` for full interface documentation. ## Deploying containers -Containers are deployed manually since [PR #46](https://github.com/OpenEuroLLM/oellm-cli/pull/46) to save costs. +Containers are deployed manually since [PR #46](https://github.com/elliot-project/elliot-cli/pull/46) to save costs. -To build and deploy them, select run workflow in [Actions](https://github.com/OpenEuroLLM/oellm-cli/actions/workflows/build-and-push-apptainer.yml). +To build and deploy them, select run workflow in [Actions](https://github.com/elliot-project/elliot-cli/actions/workflows/build-and-push-apptainer.yml). ## Troubleshooting diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md new file mode 100644 index 00000000..41a3bd5f --- /dev/null +++ b/docs/LEONARDO.md @@ -0,0 +1,60 @@ +# Leonardo HPC — Access Guide + +## 1. Setting Up an Account + +Create your account at the CINECA UserDB portal: +[https://userdb.hpc.cineca.it/](https://userdb.hpc.cineca.it/) + +Once you have set up your UserDB profile and obtained your CINECA account name, follow the steps below. + +--- + +## 2. Install `step` CLI + +### macOS (via Homebrew) + +```zsh +brew install step +``` + +> If Homebrew is not yet installed, run the following first: +> ```zsh +> /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +> ``` + +--- + +## 3. Bootstrap the CA + +```zsh +step ca bootstrap \ + --ca-url=https://sshproxy.hpc.cineca.it \ + --fingerprint 2ae1543202304d3f434bdc1a2c92eff2cd2b02110206ef06317e70c1c1735ecd +``` + +--- + +## 4. Generate an SSH Certificate + +Replace `your.name@email.com` with the email you used when registering on UserDB, and choose a name for your key file: + +```zsh +step ssh certificate your.name@email.com --provisioner cineca-hpc key_filename +``` + +--- + +## 5. Connect to Leonardo + +```zsh +ssh -i ~/.ssh/key_filename username@login07-ext.leonardo.cineca.it +``` + +Replace `username` with your CINECA account name. + +--- + +## Additional Resources + +Full instructions are also available here: +[https://iffmd.fz-juelich.de/e-hu5RBHRXG6DTgD9NVjig#Leonardo-Access-and-Usage-LAION-Open-Psi-open-sci-Ontocord-AI-openEuroLLM](https://iffmd.fz-juelich.de/e-hu5RBHRXG6DTgD9NVjig#Leonardo-Access-and-Usage-LAION-Open-Psi-open-sci-Ontocord-AI-openEuroLLM) From c8cdf4640e33c0c81559713f8f60f07a754d54dc Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Mon, 9 Mar 2026 14:03:34 +0100 Subject: [PATCH 04/44] Setting up Leonardo docs --- docs/LEONARDO.md | 118 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 104 insertions(+), 14 deletions(-) diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index 41a3bd5f..7d58a2a7 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -1,4 +1,4 @@ -# Leonardo HPC — Access Guide +# Leonardo HPC — Environment Setup Guide ## 1. Setting Up an Account @@ -9,10 +9,9 @@ Once you have set up your UserDB profile and obtained your CINECA account name, --- -## 2. Install `step` CLI - -### macOS (via Homebrew) +## 2. SSH Access (macOS) +### Install `step` CLI ```zsh brew install step ``` @@ -22,35 +21,126 @@ brew install step > /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" > ``` ---- - -## 3. Bootstrap the CA - +### Bootstrap the CA ```zsh step ca bootstrap \ --ca-url=https://sshproxy.hpc.cineca.it \ --fingerprint 2ae1543202304d3f434bdc1a2c92eff2cd2b02110206ef06317e70c1c1735ecd ``` +### Generate an SSH Certificate +Replace `your.name@email.com` with the email you used when registering on UserDB: +```zsh +step ssh certificate your.name@email.com --provisioner cineca-hpc key_filename +``` + +### Configure SSH +Add the following to `~/.ssh/config` (create it if it doesn't exist via `nano ~/.ssh/config`): +``` +Host leonardo + HostName login07-ext.leonardo.cineca.it + User your_cineca_username + IdentityFile /Users/your_mac_username/.ssh/key_filename +``` + +Set correct permissions: +```zsh +chmod 600 ~/.ssh/config +``` + +You can now connect simply with: +```zsh +ssh leonardo +``` + --- -## 4. Generate an SSH Certificate +## 3. Setting Up the Python Environment on the Cluster + +### Load Python module +```zsh +module purge +module load python/3.11.7 +``` -Replace `your.name@email.com` with the email you used when registering on UserDB, and choose a name for your key file: +### Install `uv` via a bootstrap environment +Since the cluster does not have `uv` available by default and PyPI access is restricted outside a venv, first create a temporary environment to install `uv`: +```zsh +python -m venv elliot-env +source elliot-env/bin/activate +pip install uv +``` +### Make `uv` permanently available +Copy the `uv` binary to `~/.local/bin` so it persists across all environments: ```zsh -step ssh certificate your.name@email.com --provisioner cineca-hpc key_filename +mkdir -p ~/.local/bin +cp $HOME/elliot-env/bin/uv ~/.local/bin/uv +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc +source ~/.bashrc +``` + +Verify: +```zsh +uv --version +``` + +### Install Python 3.12 via `uv` +The project requires Python 3.12, which is not available as a cluster module. `uv` can fetch it directly: +```zsh +uv python install 3.12 +``` + +### Create the project environment +```zsh +uv venv -p 3.12 elliot-venv +source elliot-venv/bin/activate +``` + +--- + +## 4. Install the Project + +### Clone the repository +```zsh +git clone https://github.com/elliot-project/elliot-cli.git +``` + +### Install in editable mode +```zsh +uv pip install -e ./elliot-cli ``` +Installing in editable mode means any changes you make to the source files are reflected immediately — no reinstall needed. + --- -## 5. Connect to Leonardo +## 5. Set HuggingFace Cache Directory + +Compute nodes have no internet access, so all models and datasets must be pre-downloaded. Set `HF_HOME` to point to your work storage: ```zsh -ssh -i ~/.ssh/key_filename username@login07-ext.leonardo.cineca.it +mkdir -p $WORK/hf_cache +echo 'export HF_HOME="$WORK/hf_cache"' >> ~/.bashrc +source ~/.bashrc ``` -Replace `username` with your CINECA account name. +--- + +## 6. Running Evaluations + +```zsh +# Run evaluations using a task group (recommended) +oellm schedule-eval \ + --models "microsoft/DialoGPT-medium,EleutherAI/pythia-160m" \ + --task_groups "open-sci-0.01" + +# Or specify individual tasks +oellm schedule-eval \ + --models "EleutherAI/pythia-160m" \ + --tasks "hellaswag,mmlu" \ + --n_shot 5 +``` --- From ea3428e66be7a768906ab135b99abed200a2e83c Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Mon, 9 Mar 2026 14:29:14 +0100 Subject: [PATCH 05/44] fix paths --- docs/LEONARDO.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index 7d58a2a7..06e51fb0 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -111,8 +111,6 @@ git clone https://github.com/elliot-project/elliot-cli.git uv pip install -e ./elliot-cli ``` -Installing in editable mode means any changes you make to the source files are reflected immediately — no reinstall needed. - --- ## 5. Set HuggingFace Cache Directory @@ -121,7 +119,7 @@ Compute nodes have no internet access, so all models and datasets must be pre-do ```zsh mkdir -p $WORK/hf_cache -echo 'export HF_HOME="$WORK/hf_cache"' >> ~/.bashrc +echo 'export HF_HOME="$HOME/hf_cache"' >> ~/.bashrc source ~/.bashrc ``` From 4cea7ae4b850476ed6360725eda95b7da06e4395 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Mon, 9 Mar 2026 16:09:12 +0100 Subject: [PATCH 06/44] fix HF_HOME --- docs/LEONARDO.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index 06e51fb0..aa181263 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -118,8 +118,8 @@ uv pip install -e ./elliot-cli Compute nodes have no internet access, so all models and datasets must be pre-downloaded. Set `HF_HOME` to point to your work storage: ```zsh -mkdir -p $WORK/hf_cache -echo 'export HF_HOME="$HOME/hf_cache"' >> ~/.bashrc +mkdir -p $HOME/hf_cache +export HF_HOME="$HOME/hf_cache" source ~/.bashrc ``` From 2eb5a1f7b915a34033dacf8ced2aee7624d84bc6 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Mon, 9 Mar 2026 21:52:20 +0100 Subject: [PATCH 07/44] bug fixes --- README.md | 38 +++++++++++---- oellm/main.py | 78 ++++++++++++++++++++++++++----- oellm/resources/task-groups.yaml | 80 +++++++++++++++++++++++++++++++- oellm/resources/template.sbatch | 16 +++---- pyproject.toml | 2 +- requirements-venv.txt | 2 +- 6 files changed, 185 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f59c2d9d..215e28c5 100644 --- a/README.md +++ b/README.md @@ -64,14 +64,39 @@ Super groups: ### Image -| Group | Benchmarks | Engine | +| Group | Benchmark | Engine | |---|---|---| -| `image-vqa` | VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista | lmms-eval | - -Image evaluation requires a container or venv with `lmms-eval` installed. Install the optional dependency: +| `image-vqa` | All 8 benchmarks combined | lmms-eval | +| `image-vqav2` | VQAv2 | lmms-eval | +| `image-mmbench` | MMBench | lmms-eval | +| `image-mmmu` | MMMU | lmms-eval | +| `image-chartqa` | ChartQA | lmms-eval | +| `image-docvqa` | DocVQA | lmms-eval | +| `image-textvqa` | TextVQA | lmms-eval | +| `image-ocrbench` | OCRBench | lmms-eval | +| `image-mathvista` | MathVista | lmms-eval | + +Image evaluation requires a venv with `lmms-eval` installed (see [docs/VENV.md](docs/VENV.md)). The lmms-eval adapter class (`llava_hf`, `qwen2_5_vl`, etc.) is auto-detected from the model name — no extra configuration needed. ```bash -pip install "oellm[image]" +# Run all 8 image benchmarks at once +oellm schedule-eval \ + --models "llava-hf/llava-1.5-7b-hf" \ + --task_groups "image-vqa" \ + --venv_path ~/elliot-venv + +# Smoke-test a single benchmark (fast, use --limit for a few samples) +oellm schedule-eval \ + --models "llava-hf/llava-1.5-7b-hf" \ + --task_groups "image-mathvista" \ + --venv_path ~/elliot-venv \ + --limit 10 + +# Mix image and text benchmarks in one submission +oellm schedule-eval \ + --models "llava-hf/llava-1.5-7b-hf" \ + --task_groups "image-mmbench,open-sci-0.01" \ + --venv_path ~/elliot-venv ``` ```bash @@ -83,9 +108,6 @@ oellm schedule-eval --models "model-name" --task_groups "belebele-eu-5-shot,glob # Use a super group oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" - -# Image evaluation -oellm schedule-eval --models "model-name" --task_groups "image-vqa" ``` ## SLURM Overrides diff --git a/oellm/main.py b/oellm/main.py index 915c029b..b8c13bb7 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -39,6 +39,34 @@ class EvaluationJob: eval_suite: str +def _detect_lmms_model_type(model_path: str) -> str: + """Detect the lmms-eval adapter class name from a model path or HF repo name. + + lmms-eval requires --model (e.g. llava_hf, qwen2_5_vl). + This is inferred from the model name so users never need to set it manually. + To add support for a new model family, add a pattern here. + """ + name = str(model_path).lower() + if "qwen2.5-vl" in name or "qwen2_5_vl" in name or "qwen2.5vl" in name: + return "qwen2_5_vl" + if "qwen2-vl" in name or "qwen2_vl" in name: + return "qwen2_vl" + if "llava" in name: + return "llava_hf" + if "internvl" in name: + return "internvl2" + if "idefics" in name: + return "idefics3" + if "minicpm" in name: + return "minicpm_v" + if "qwen" in name: + return "qwen_vl" + raise ValueError( + f"Cannot auto-detect lmms-eval adapter class from model path '{model_path}'. " + "Add your model family to _detect_lmms_model_type() in main.py." + ) + + @capture_third_party_output_from_kwarg("verbose") def schedule_evals( models: str | None = None, @@ -193,6 +221,14 @@ def schedule_evals( ) ) + # For lmms_eval jobs, encode the adapter class in eval_suite as "lmms_eval:". + # This makes LMMS_MODEL_TYPE completely transparent — users never set it manually. + for job in expanded_eval_jobs: + if job.eval_suite == "lmms_eval": + adapter = _detect_lmms_model_type(str(job.model_path)) + job.eval_suite = f"lmms_eval:{adapter}" + logging.debug(f"lmms-eval adapter for {job.model_path}: {adapter}") + if not skip_checks: hub_models: set[str | Path] = { job.model_path @@ -471,19 +507,23 @@ def _first_matching_prefix( val, key = _first_matching_prefix(result_dict, metric.split(",")[0]) if val is not None: return val, key + + # Last resort: pick the first numeric non-stderr value (catches lmms-eval + # benchmarks with non-standard metric names like mme_cognition_score) + for k, v in result_dict.items(): + if isinstance(v, (int, float)) and "stderr" not in k and k not in ("alias", " ", ""): + return float(v), k return None, None results_path = Path(results_dir) if not results_path.exists(): raise ValueError(f"Results directory does not exist: {results_dir}") - # Check if we need to look in a 'results' subdirectory - if (results_path / "results").exists() and (results_path / "results").is_dir(): - # User passed the top-level directory, look in results subdirectory - json_files = list((results_path / "results").glob("*.json")) - else: - # User passed the results directory directly - json_files = list(results_path.glob("*.json")) + # lm-eval writes flat JSON files: results/.json + # lmms-eval writes nested dirs: results/.json//_results.json + # rglob("*.json") + is_file() finds both without breaking backward compat. + search_root = (results_path / "results") if (results_path / "results").is_dir() else results_path + json_files = [p for p in search_root.rglob("*.json") if p.is_file()] if not json_files: logging.warning(f"No JSON files found in {results_dir}") @@ -519,6 +559,14 @@ def _first_matching_prefix( results = data.get("results", {}) n_shot_data = data.get("n-shot", {}) + # lmms-eval has no "n-shot" dict; fall back to per-task config "num_fewshot" + if not n_shot_data: + for _task, _cfg in data.get("configs", {}).items(): + if isinstance(_cfg, dict): + shot = _cfg.get("num_fewshot") + if shot is not None: + n_shot_data[_task] = shot + # Infer a global n_shot if exactly one unique value exists in this JSON global_n_shot = None try: @@ -617,6 +665,10 @@ def _first_matching_prefix( if n_shot == "unknown" and global_n_shot is not None: n_shot = global_n_shot + # Skip lmms-eval parent task placeholders (no numeric metrics, just alias) + if set(task_results.keys()) <= {"alias", " ", ""}: + continue + # Get the primary metric (usually acc, acc_norm) performance, metric_name = _resolve_metric(task_name, task_results) @@ -635,11 +687,13 @@ def _first_matching_prefix( } ) else: - # Debug: log cases where we have a task but no performance metric - if verbose: - logging.debug( - f"No performance metric found for {model_name} | {task_name} | n_shot={n_shot} in {json_file.name}" - ) + # Log missing metrics — for lmms-eval tasks this often means + # llm_as_judge_eval is null (no judge LLM configured) or the + # metric key is not yet listed in task_metrics in task-groups.yaml + logging.warning( + f"No numeric metric for '{task_name}' in {json_file.name} " + f"— value may be null (LLM judge not configured?) or metric key missing from task_metrics" + ) if not rows and not check: logging.warning("No results extracted from JSON files") diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 1757df9e..361b35db 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -12,13 +12,18 @@ task_metrics: piqa: acc_norm # lmms-eval image benchmark metrics vqav2_val_all: vqa_score + mme: mme_cognition_score mmbench_en_dev: acc mmmu_val: acc chartqa: relaxed_accuracy docvqa_val: anls textvqa_val: acc ocrbench: score - mathvista_testmini: acc + # MathVista uses LLM-as-judge; null without judge LLM configured + mathvista_testmini: llm_as_judge_eval + mathvista_testmini_cot: llm_as_judge_eval + mathvista_testmini_format: llm_as_judge_eval + mathvista_testmini_solution: llm_as_judge_eval task_groups: open-sci-0.01: @@ -325,6 +330,79 @@ task_groups: - task: mathvista_testmini dataset: AI4Math/MathVista + # ── Individual Image Benchmarks (single-task groups for targeted runs) ────── + image-vqav2: + description: "VQAv2 visual question answering via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: vqav2_val_all + dataset: HuggingFaceM4/VQAv2 + + image-mmbench: + description: "MMBench multi-modal benchmark via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mmbench_en_dev + dataset: HuggingFaceM4/MMBench_00 + + image-mmmu: + description: "MMMU massive multi-discipline multimodal understanding via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mmmu_val + dataset: MMMU/MMMU + + image-chartqa: + description: "ChartQA chart question answering via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: chartqa + dataset: HuggingFaceM4/ChartQA + + image-docvqa: + description: "DocVQA document visual question answering via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: docvqa_val + dataset: eliolio/docvqa + + image-textvqa: + description: "TextVQA text-based visual question answering via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: textvqa_val + dataset: facebook/textvqa + + image-ocrbench: + description: "OCRBench optical character recognition benchmark via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: ocrbench + dataset: echo840/OCRBench + + image-mme: + description: "MME multimodal evaluation benchmark via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mme + dataset: lmms-lab/MME + + image-mathvista: + description: "MathVista mathematical reasoning in visual contexts via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mathvista_testmini + dataset: AI4Math/MathVista + super_groups: oellm-multilingual: description: "Combined Belebele EU set plus multilingual benchmarks" diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 98b16592..25d5936a 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -91,7 +91,8 @@ do fi fi - suite_normalized=$(echo "$eval_suite" | tr '[:upper:]' '[:lower:]') + # Strip optional ":adapter" suffix before matching (e.g. "lmms_eval:qwen2_5_vl" -> "lmms_eval") + suite_normalized=$(echo "${{eval_suite%%:*}}" | tr '[:upper:]' '[:lower:]') # Helper function to run Python commands in the appropriate environment run_python() {{ @@ -159,16 +160,15 @@ do ;; lmms_eval|lmms-eval) # lmms-eval is used for image/video/audio benchmarks. - # Model type defaults to llava_hf (HuggingFace LLaVA-style VLMs). - # Override via LMMS_MODEL_TYPE in clusters.yaml or --slurm_template_var. - LMMS_MODEL_TYPE="${{LMMS_MODEL_TYPE:-llava_hf}}" + # Adapter class is encoded in eval_suite as "lmms_eval:" by the oellm CLI. + _lmms_adapter="${{eval_suite#*:}}" OUTPUT_JSON="{evals_dir}/$(openssl rand -hex 5).json" if [ -n "$VENV_PATH" ]; then source "$VENV_PATH/bin/activate" python -m lmms_eval \ - --model "$LMMS_MODEL_TYPE" \ - --model_args "pretrained=$model_path" \ + --model "$_lmms_adapter" \ + --model_args "pretrained=$model_path,device_map=auto" \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ @@ -178,8 +178,8 @@ do --bind $BIND_PATHS \ $EVAL_SIF_PATH \ python -m lmms_eval \ - --model "$LMMS_MODEL_TYPE" \ - --model_args "pretrained=$model_path" \ + --model "$_lmms_adapter" \ + --model_args "pretrained=$model_path,device_map=auto" \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ diff --git a/pyproject.toml b/pyproject.toml index 36e10d98..11053edb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dev = [ "pre-commit", ] image = [ - "lmms-eval>=0.2.4", + "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", ] [project.scripts] diff --git a/requirements-venv.txt b/requirements-venv.txt index c55e4ddd..5a6c44a5 100644 --- a/requirements-venv.txt +++ b/requirements-venv.txt @@ -8,7 +8,7 @@ datasets<4.0.0 # lmms-eval: image/video/audio evaluation engine (required for image-vqa task group) # lmms-eval is compatible with datasets<4.0.0; install alongside lm-eval. -lmms-eval>=0.2.4 +lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git # lighteval must be installed separately as a uv tool to avoid datasets version conflict: # UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ From 5908bb6e8cf2ab35d7d053d66fac55e4be489b10 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Tue, 24 Mar 2026 10:08:02 +0100 Subject: [PATCH 08/44] [Contrib][Custom Benchmark] Adding Region Reasoner benchmark Integration of RegionReasoner: Region-Grounded Multi-Round Visual Reasoning (ICLR 2026) benchmark: https://arxiv.org/pdf/2602.03733. Code is available here: https://github.com/lmsdss/RegionReasoner. --- .github/workflows/ci.yml | 2 +- oellm/contrib/CONTRIBUTING.md | 271 ++++++++++ oellm/contrib/__init__.py | 3 + oellm/contrib/dispatch.py | 91 ++++ oellm/contrib/region_reasoner/README.md | 133 +++++ oellm/contrib/region_reasoner/__init__.py | 2 + oellm/contrib/region_reasoner/adapter.py | 32 ++ oellm/contrib/region_reasoner/metrics.py | 151 ++++++ oellm/contrib/region_reasoner/suite.py | 275 ++++++++++ oellm/contrib/region_reasoner/task.py | 50 ++ oellm/core/base_metric.py | 4 +- oellm/core/base_model_adapter.py | 21 +- oellm/core/base_task.py | 132 ++++- oellm/main.py | 118 ++-- oellm/registry.py | 131 +++++ oellm/resources/template.sbatch | 28 +- oellm/task_groups.py | 144 +++-- oellm/utils.py | 58 +- pyproject.toml | 3 + tests/test_base_interfaces.py | 5 +- tests/test_collect_results.py | 7 +- tests/test_image_task_groups.py | 12 +- tests/test_region_reasoner.py | 630 ++++++++++++++++++++++ tests/test_registry.py | 62 +++ tests/test_schedule_evals.py | 1 + 25 files changed, 2225 insertions(+), 141 deletions(-) create mode 100644 oellm/contrib/CONTRIBUTING.md create mode 100644 oellm/contrib/__init__.py create mode 100644 oellm/contrib/dispatch.py create mode 100644 oellm/contrib/region_reasoner/README.md create mode 100644 oellm/contrib/region_reasoner/__init__.py create mode 100644 oellm/contrib/region_reasoner/adapter.py create mode 100644 oellm/contrib/region_reasoner/metrics.py create mode 100644 oellm/contrib/region_reasoner/suite.py create mode 100644 oellm/contrib/region_reasoner/task.py create mode 100644 oellm/registry.py create mode 100644 tests/test_region_reasoner.py create mode 100644 tests/test_registry.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8636b26e..7abed16d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,4 +66,4 @@ jobs: run: uv sync --extra dev - name: Run tests - run: uv run pytest tests/ -v --ignore=tests/integration/ + run: uv run pytest tests/ -v --ignore=tests/integration/ --ignore=tests/test_datasets.py diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md new file mode 100644 index 00000000..61947b57 --- /dev/null +++ b/oellm/contrib/CONTRIBUTING.md @@ -0,0 +1,271 @@ +# Integrating Custom Benchmarks + +There are two ways to add a benchmark to elliot-cli. + +--- + +## Path 1 — Benchmark already in lm-eval / lighteval / lmms-eval + +Add a task group entry to `oellm/resources/task-groups.yaml`. + +```yaml +task_metrics: + my_task_name: acc + +task_groups: + my-benchmark: + description: "My benchmark via lmms-eval" + suite: lmms_eval # or lm-eval-harness / lighteval + n_shots: [0] + tasks: + - task: my_task_name + dataset: org/my-hf-dataset +``` + +```bash +oellm schedule-eval \ + --models org/MyModel \ + --task_groups my-benchmark \ + --venv_path ~/elliot-venv +``` + +Supported `suite` values: + +| `suite` | Framework | +|---|---| +| `lm-eval-harness` | [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) | +| `lighteval` | [lighteval](https://github.com/huggingface/lighteval) | +| `lmms_eval` | [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) | + +--- + +## Path 2 — Custom benchmark (new contrib suite) + +Use this when the benchmark has its own inference script, custom metrics, or +requires multi-GPU sharding. + +Drop files into `oellm/contrib/my_suite/`. No changes to any core file. + +``` +oellm/contrib/my_suite/ +├── __init__.py +├── suite.py +├── task.py +├── adapter.py +├── metrics.py +└── README.md +``` + +See `oellm/contrib/region_reasoner/` as a complete reference. + +--- + +### Step 1 — task.py + +```python +from oellm.core.base_task import BaseTask + + +class MyTask(BaseTask): + # Required + @property + def name(self) -> str: + return "my_task_name" + + @property + def suite(self) -> str: + return "my_suite" + + @property + def n_shots(self) -> list[int]: + return [0] + + # Optional + @property + def task_group_name(self) -> str: + return "my-benchmark" # default: name with _ replaced by - + + @property + def description(self) -> str: + return "My benchmark on dataset X." + + @property + def primary_metric(self) -> str | None: + return "my_primary_metric" + + @property + def hf_models(self) -> list[str]: + return ["org/auxiliary-model"] + + @property + def hf_dataset_files(self) -> list[dict]: + return [ + { + "repo_id": "org/my-dataset", + "patterns": ["data/test.json", "data/images/*"], + } + ] +``` + +`hf_models` and `hf_dataset_files` are downloaded on the login node before +SLURM submission. Compute nodes run with `HF_HUB_OFFLINE=1`. Use `patterns` +to download only the files you need from large repos. + +--- + +### Step 2 — adapter.py + +```python +from pathlib import Path +from oellm.core.base_model_adapter import BaseModelAdapter + + +class MyModelAdapter(BaseModelAdapter): + def __init__(self, model_path: str) -> None: + self._path = model_path + + @property + def model_path(self) -> str: + return self._path + + def to_lm_eval_args(self) -> str: + return f"pretrained={self._path},trust_remote_code=True" + + def to_lmms_eval_args(self) -> str: + return f"pretrained={self._path}" + + def to_contrib_flags(self) -> str | None: + """Return a model-type suffix passed to run() as model_flags, or None.""" + name = Path(self._path).name.lower() + if "mymodel_v2" in name: + return "backend_b" + return "backend_a" +``` + +The string returned by `to_contrib_flags()` is appended to `eval_suite` in +`jobs.csv` as `"my_suite:backend_a"` and passed to `run()` as `model_flags`. +Use it to select between different inference backends for the same benchmark. + +--- + +### Step 3 — suite.py + +```python +from pathlib import Path +from oellm.contrib.my_suite.task import MyTask + +SUITE_NAME = "my_suite" + +TASK_GROUPS: dict = MyTask.to_task_groups_dict() + +CLUSTER_ENV_VARS = ["MY_DATA_DIR"] + + +def detect_model_flags(model_path: str) -> str | None: + from oellm.contrib.my_suite.adapter import MyModelAdapter + return MyModelAdapter(model_path).to_contrib_flags() + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Run the evaluation and write results to output_path. + + output_path must be a lmms-eval-compatible JSON: + + { + "model_name_or_path": "", + "results": { + "": {"metric_a": 0.42, "metric_b": 0.38} + }, + "configs": { + "": {"num_fewshot": } + } + } + """ + data_dir = env["MY_DATA_DIR"] + # ... run inference, compute metrics, write output_path ... + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Try to parse data as output from this suite. + + Return (model_id, task_name, n_shot, {metric: value}) or None. + """ + results = data.get("results", {}) + for task_name, task_results in results.items(): + if task_name.startswith("my_task_") and "my_primary_metric" in task_results: + model_id = data.get("model_name_or_path", "unknown") + n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) + return model_id, task_name, int(n_shot), task_results + return None +``` + +`CLUSTER_ENV_VARS` are validated by `dispatch.py` before `run()` is called. + +--- + +### Step 4 — metrics.py + +```python +from oellm.core.base_metric import BaseMetric + + +class MyMetric(BaseMetric): + @property + def name(self) -> str: + return "my_metric" + + def compute(self, predictions: list[str], references: list[str]) -> float: + import json + correct = sum( + json.loads(p) == json.loads(r) + for p, r in zip(predictions, references) + ) + return correct / len(predictions) if predictions else 0.0 +``` + +`compute()` receives predictions and references as JSON-serialized strings. + +--- + +### Step 5 — clusters.yaml + +Add cluster-specific paths to `oellm/resources/clusters.yaml`: + +```yaml +my-cluster: + MY_DATA_DIR: "/path/to/benchmark/data" + MY_SUITE_NUM_GPUS: "4" +``` + +--- + +### Step 6 — tests + +Add `tests/test_my_suite.py`. Cover at minimum: + +- `SUITE_NAME`, `CLUSTER_ENV_VARS`, `TASK_GROUPS` structure +- `MyTask` properties: `name`, `suite`, `n_shots`, `primary_metric`, `hf_models`, `hf_dataset_files`, `task_group_name` +- `MyTask.to_task_groups_dict()` output structure +- `MyModelAdapter.to_contrib_flags()` for expected model name patterns +- `detect_model_flags()` return values +- `parse_results()` for a matching and a non-matching JSON +- Metric `compute()` correctness +- Task group expansion: correct `(task, n_shot, suite)` tuples +- Dry-run `schedule_evals()` produces SBATCH with `oellm.contrib.dispatch` + +See `tests/test_region_reasoner.py` as a reference. + +--- + +## Notes + +- Point `HF_HOME` to a filesystem with sufficient space. Home directories on + HPC clusters typically have small quotas (50 GB or less). diff --git a/oellm/contrib/__init__.py b/oellm/contrib/__init__.py new file mode 100644 index 00000000..310d48d8 --- /dev/null +++ b/oellm/contrib/__init__.py @@ -0,0 +1,3 @@ +# Contrib plugin packages live here. +# Each sub-package must contain a suite.py exposing the plugin protocol. +# See oellm/registry.py for the full protocol documentation. diff --git a/oellm/contrib/dispatch.py b/oellm/contrib/dispatch.py new file mode 100644 index 00000000..de4d4b70 --- /dev/null +++ b/oellm/contrib/dispatch.py @@ -0,0 +1,91 @@ +"""SLURM-side entry point for contrib suite evaluation. + +Called from template.sbatch's ``*)`` catch-all case as:: + + python -m oellm.contrib.dispatch \\ + --suite "region_reasoner:vision_reasoner" \\ + --model_path "/path/to/model" \\ + --task "regionreasoner_refcocog" \\ + --n_shot 0 \\ + --output_path "/evals/dir/abc123.json" + +The suite name may include a model-flags suffix separated by ``:``, e.g. +``region_reasoner:vision_reasoner``. The suffix is passed to ``suite.run()`` +as ``model_flags``. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +from pathlib import Path + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Dispatch evaluation to a registered contrib suite." + ) + p.add_argument("--suite", required=True, help="Suite name (optionally :model_flags)") + p.add_argument("--model_path", required=True) + p.add_argument("--task", required=True) + p.add_argument("--n_shot", required=True, type=int) + p.add_argument("--output_path", required=True, type=Path) + return p.parse_args(argv) + + +def main(argv: list[str] | None = None) -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + + args = _parse_args(argv) + + if ":" in args.suite: + suite_name, model_flags = args.suite.split(":", 1) + else: + suite_name, model_flags = args.suite, None + + from oellm import registry + + try: + mod = registry.get_suite(suite_name) + except KeyError as exc: + logging.error(str(exc)) + sys.exit(1) + + missing_vars = [] + for var in getattr(mod, "CLUSTER_ENV_VARS", []): + if not os.environ.get(var): + missing_vars.append(var) + if missing_vars: + logging.error( + "Contrib suite %r requires environment variables that are not set: %s\n" + "Add them to clusters.yaml for this cluster.", + suite_name, + ", ".join(missing_vars), + ) + sys.exit(1) + + args.output_path.parent.mkdir(parents=True, exist_ok=True) + + logging.info( + "Dispatching to suite %r (model_flags=%r): %s | %s | n_shot=%d", + suite_name, + model_flags, + args.model_path, + args.task, + args.n_shot, + ) + + mod.run( + model_path=args.model_path, + task=args.task, + n_shot=args.n_shot, + output_path=args.output_path, + model_flags=model_flags, + env=dict(os.environ), + ) + + +if __name__ == "__main__": + main() diff --git a/oellm/contrib/region_reasoner/README.md b/oellm/contrib/region_reasoner/README.md new file mode 100644 index 00000000..b9cc90c4 --- /dev/null +++ b/oellm/contrib/region_reasoner/README.md @@ -0,0 +1,133 @@ +# RegionReasoner Benchmark + +Multi-turn region grounding benchmark on RefCOCOg. Evaluates a model's ability +to locate and segment objects described in multi-turn conversations. + +**Metrics:** gIoU, cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9 + +--- + +## Prerequisites + +### 1. Clone RegionReasoner + +The benchmark relies on the inference script +`test/evaluation/evaluation_multi_segmentation.py` and the model wrapper +`test/vision_reasoner/` from the RegionReasoner repository. These are **not +packaged** — the platform calls them directly as a subprocess, so the repo +must be present on the cluster filesystem. + +```bash +git clone https://github.com/lmsdss/RegionReasoner \ + /path/to/RegionReasoner +``` + +### 2. Configure clusters.yaml + +Add the following to your cluster entry in `oellm/resources/clusters.yaml`: + +```yaml +my-cluster: + ... + HF_HOME: "/path/to/large/filesystem/huggingface" # must have ~30 GB free + REGION_REASONER_DIR: "/path/to/RegionReasoner" + REGION_REASONER_NUM_GPUS: "4" # optional, default: 4 +``` + +> **`HF_HOME`** must point to a filesystem with at least **30 GB** of free +> space. On CINECA Leonardo, use the work filesystem +> (`/leonardo_work//huggingface`), not the home filesystem (50 GB +> quota, fills up quickly). + +### 3. Install dependencies in your venv + +```bash +# PyTorch — match the CUDA version available on your cluster +pip install torch==2.5.1 --index-url https://download.pytorch.org/whl/cu121 + +# Matching torchvision +pip install torchvision==0.20.1 --index-url https://download.pytorch.org/whl/cu121 + +# flash-attn pre-built wheel (no compilation needed) +wget https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl +pip install flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl + +# HEIF image support +pip install pi-heif +``` + +> **flash-attn note:** The pre-built wheel above is for Python 3.12, CUDA 12.x, +> torch 2.5.1. If your configuration differs, find the matching wheel at +> https://github.com/Dao-AILab/flash-attention/releases + +### 4. What gets auto-downloaded + +When you run `oellm schedule-eval`, the platform automatically pre-downloads +the following on the login node (before SLURM submission, so compute nodes do +not need internet access): + +| Asset | HF repo | Size | +|---|---|---| +| TaskRouter-1.5B | `Ricky06662/TaskRouter-1.5B` | ~3 GB | +| SAM2 | `facebook/sam2-hiera-large` | ~1 GB | +| Test JSON | `lmsdss/regionreasoner_test_data` `raw/refcocog_multi_turn.json` | ~26 GB | +| Test images | `lmsdss/regionreasoner_test_data` `raw/refcocog_test_multi_bbox_images/*` | ~1 GB | + +All assets are cached under `$HF_HOME/hub`. + +--- + +## Running + +```bash +oellm schedule-eval \ + --models lmsdss/RegionReasoner-7B \ + --task_groups region-reasoner \ + --venv_path ~/elliot-venv +``` + +### Collecting results + +```bash +oellm collect-results \ + --eval_output_dir /path/to/evals \ + --output results.csv +``` + +The primary metric in the CSV is **gIoU**. + +--- + +## Evaluating a different model + +The inference script supports multiple model types via the `--model` flag, +which is detected automatically from the model name. To evaluate a different +model, just pass it to `--models`: + +```bash +oellm schedule-eval \ + --models Qwen/Qwen2.5-VL-7B-Instruct \ + --task_groups region-reasoner \ + --venv_path ~/elliot-venv +``` + +The model type is resolved as follows: + +| Model name pattern | `--model` flag | +|---|---| +| `*regionreasoner*` / `*region_reasoner*` | `vision_reasoner` | +| `*qwen2*` | `qwen2` | +| `*qwen*` | `qwen` | +| anything else | `vision_reasoner` (default) | + +To evaluate multiple models in one go: + +```bash +oellm schedule-eval \ + --models lmsdss/RegionReasoner-7B Qwen/Qwen2.5-VL-7B-Instruct \ + --task_groups region-reasoner \ + --venv_path ~/elliot-venv +``` + +> If your model name does not match any pattern above and requires a specific +> `--model` flag, extend `detect_model_flags()` in `suite.py`. diff --git a/oellm/contrib/region_reasoner/__init__.py b/oellm/contrib/region_reasoner/__init__.py new file mode 100644 index 00000000..f581d4ea --- /dev/null +++ b/oellm/contrib/region_reasoner/__init__.py @@ -0,0 +1,2 @@ +# RegionReasoner contrib package. +# See suite.py for the plugin protocol implementation. diff --git a/oellm/contrib/region_reasoner/adapter.py b/oellm/contrib/region_reasoner/adapter.py new file mode 100644 index 00000000..b849d708 --- /dev/null +++ b/oellm/contrib/region_reasoner/adapter.py @@ -0,0 +1,32 @@ +"""RegionReasoner model adapter.""" + +from pathlib import Path + +from oellm.core.base_model_adapter import BaseModelAdapter + + +class RegionReasonerModelAdapter(BaseModelAdapter): + """Translates a model path into eval-engine argument strings.""" + + def __init__(self, model_path: str) -> None: + self._path = model_path + + @property + def model_path(self) -> str: + return self._path + + def to_lm_eval_args(self) -> str: + return f"pretrained={self._path},trust_remote_code=True" + + def to_lmms_eval_args(self) -> str: + return f"pretrained={self._path}" + + def to_contrib_flags(self) -> str | None: + name = Path(self._path).name.lower() + if "regionreasoner" in name or "region_reasoner" in name: + return "vision_reasoner" + if "qwen2" in name: + return "qwen2" + if "qwen" in name: + return "qwen" + return "vision_reasoner" diff --git a/oellm/contrib/region_reasoner/metrics.py b/oellm/contrib/region_reasoner/metrics.py new file mode 100644 index 00000000..5b84be45 --- /dev/null +++ b/oellm/contrib/region_reasoner/metrics.py @@ -0,0 +1,151 @@ +"""Region-grounding benchmark metrics (model-agnostic). + +:class:`BaseMetric` subclasses that compute region-grounding metrics from +per-sample inference results. These are used by ``suite._aggregate_shards()`` +to score any model's predictions. + +Input format +------------ +Each sample is a JSON-serialised dict containing pre-computed fields from the +inference script:: + + { + "intersection": 12345, + "union": 23456, + "bbox_iou": 0.73 + } + +``predictions`` passed to ``compute()`` are ``list[str]`` — one JSON string +per sample. ``references`` are unused (empty strings) since ground truth is +already folded into the intersection/union computation by the inference script. + +Metrics +------- +- **GIoU**: mean of per-sample mask IoU (intersection / union). +- **CIoU**: cumulative IoU — sum of all intersections / sum of all unions. +- **BboxAP**: fraction of samples where bbox IoU > 0.5. +- **PassRate**: fraction of samples where mask IoU > *threshold*. +""" + +from __future__ import annotations + +import json + +from oellm.core.base_metric import BaseMetric + + +def _parse_sample(s: str) -> dict | None: + """Parse a JSON-serialised sample dict. Returns None on failure.""" + if not s or s.strip() in ("null", "none", ""): + return None + try: + val = json.loads(s) + if isinstance(val, dict): + return val + except (json.JSONDecodeError, ValueError, TypeError): + pass + return None + + +def _mask_iou(sample: dict) -> float: + """Compute mask IoU from pre-computed intersection and union.""" + intersection = sample.get("intersection", 0) + union = sample.get("union", 0) + if union <= 0: + return 0.0 + return intersection / union + + +class GIoU(BaseMetric): + """Mean per-sample mask IoU (gIoU as reported in the RegionReasoner paper). + + Formula: ``mean(intersection_i / union_i)`` over all samples. + """ + + @property + def name(self) -> str: + return "gIoU" + + def compute(self, predictions: list[str], references: list[str]) -> float: + if not predictions: + return 0.0 + ious = [] + for s in predictions: + sample = _parse_sample(s) + ious.append(_mask_iou(sample) if sample else 0.0) + return sum(ious) / len(ious) + + +class CIoU(BaseMetric): + """Cumulative IoU (cIoU as reported in the RegionReasoner paper). + + Formula: ``sum(all intersections) / sum(all unions)``. + """ + + @property + def name(self) -> str: + return "cIoU" + + def compute(self, predictions: list[str], references: list[str]) -> float: + total_intersection = 0.0 + total_union = 0.0 + for s in predictions: + sample = _parse_sample(s) + if sample is None: + continue + total_intersection += sample.get("intersection", 0) + total_union += sample.get("union", 0) + if total_union <= 0.0: + return 0.0 + return total_intersection / total_union + + +class BboxAP(BaseMetric): + """Binarised bounding-box AP at IoU threshold 0.5. + + Uses the pre-computed ``bbox_iou`` field from the inference script. + Formula: ``mean(bbox_iou_i > 0.5)``. + """ + + @property + def name(self) -> str: + return "bbox_AP" + + def compute(self, predictions: list[str], references: list[str]) -> float: + if not predictions: + return 0.0 + hits = 0 + for s in predictions: + sample = _parse_sample(s) + if sample and sample.get("bbox_iou", 0) > 0.5: + hits += 1 + return hits / len(predictions) + + +class PassRate(BaseMetric): + """Fraction of samples where mask IoU exceeds a configurable threshold. + + Args: + threshold: IoU threshold in (0, 1). Common values: 0.3, 0.5, 0.7, 0.9. + """ + + def __init__(self, threshold: float = 0.5) -> None: + if not 0 < threshold < 1: + raise ValueError(f"threshold must be in (0, 1), got {threshold}") + self._threshold = threshold + + @property + def name(self) -> str: + t = self._threshold + label = f"{t:.10g}" + return f"pass_rate_{label}" + + def compute(self, predictions: list[str], references: list[str]) -> float: + if not predictions: + return 0.0 + hits = 0 + for s in predictions: + sample = _parse_sample(s) + if sample and _mask_iou(sample) > self._threshold: + hits += 1 + return hits / len(predictions) diff --git a/oellm/contrib/region_reasoner/suite.py b/oellm/contrib/region_reasoner/suite.py new file mode 100644 index 00000000..90331c10 --- /dev/null +++ b/oellm/contrib/region_reasoner/suite.py @@ -0,0 +1,275 @@ +"""RegionReasoner contrib suite — plugin protocol implementation. + +This module follows the plugin protocol defined in ``oellm/registry.py``. +It is the reference implementation for custom benchmark integration. + +Cluster setup +------------- +The following environment variables must be set in ``clusters.yaml`` (or the +cluster's module/profile system) before using the ``region-reasoner`` task group: + +``REGION_REASONER_DIR`` + Absolute path to a local clone of the RegionReasoner repository + (https://github.com/lmsdss/RegionReasoner). The eval scripts + ``test/evaluation/evaluation_multi_segmentation.py`` must be present. + +``REGION_REASONER_TEST_JSON`` *(optional)* + Absolute path to the test JSON file (``refcocog_multi_turn.json``). + If not set, the file is downloaded automatically from + ``lmsdss/regionreasoner_test_data`` on the HF Hub before inference. + +``REGION_REASONER_NUM_GPUS`` *(optional, default: 4)* + Number of GPU shards to use for parallel inference. Should match the + number of GPUs available on the compute node. + +Output format +------------- +``run()`` writes a **lmms-eval-compatible JSON** file so that +``oellm.main.collect_results()`` works without modification:: + + { + "model_name_or_path": "", + "results": { + "regionreasoner_refcocog": { + "gIoU": 0.42, "cIoU": 0.45, "bbox_AP": 0.38, + "pass_rate_0.3": 0.71, "pass_rate_0.5": 0.55, + "pass_rate_0.7": 0.31, "pass_rate_0.9": 0.08 + } + }, + "configs": { + "regionreasoner_refcocog": {"num_fewshot": 0} + } + } +""" + +from __future__ import annotations + +import json +import logging +import subprocess +import tempfile +from pathlib import Path + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Plugin protocol: required constants +# --------------------------------------------------------------------------- + +SUITE_NAME = "region_reasoner" + +CLUSTER_ENV_VARS = [ + "REGION_REASONER_DIR", +] + +from oellm.contrib.region_reasoner.task import RegionReasonerTask # noqa: E402 + +TASK_GROUPS: dict = RegionReasonerTask.to_task_groups_dict() + +# --------------------------------------------------------------------------- +# Plugin protocol: optional — model-flag detection +# --------------------------------------------------------------------------- + + +def detect_model_flags(model_path: str) -> str | None: + """Delegate to RegionReasonerModelAdapter.to_contrib_flags().""" + from oellm.contrib.region_reasoner.adapter import RegionReasonerModelAdapter + + return RegionReasonerModelAdapter(model_path).to_contrib_flags() + + +# --------------------------------------------------------------------------- +# Plugin protocol: required — run evaluation +# --------------------------------------------------------------------------- + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Execute the RegionReasoner evaluation and write results to *output_path*. + + This function: + + 1. Runs ``evaluation_multi_segmentation.py`` in parallel GPU shards. + 2. Computes gIoU, cIoU, bbox_AP, and pass_rate at four thresholds using + the :mod:`oellm.contrib.region_reasoner.metrics` implementations. + 3. Writes a lmms-eval-compatible JSON to *output_path*. + + Args: + model_path: Path or HF repo ID of the model checkpoint. + task: Task name (e.g. ``"regionreasoner_refcocog"``). + n_shot: Number of few-shot examples (always 0 for this benchmark). + output_path: Where to write the results JSON. + model_flags: Model type string (e.g. ``"vision_reasoner"``). + env: Environment variables dict (from ``os.environ``). + """ + rr_dir = env.get("REGION_REASONER_DIR", "") + num_gpus = int(env.get("REGION_REASONER_NUM_GPUS", "4")) + num_parts = int(env.get("REGION_REASONER_NUM_PARTS", str(num_gpus))) + model_type = model_flags or "vision_reasoner" + + if not rr_dir: + raise RuntimeError( + "REGION_REASONER_DIR must be set. Add it to clusters.yaml for this cluster." + ) + + test_json = env.get("REGION_REASONER_TEST_JSON") + if not test_json: + from huggingface_hub import snapshot_download + + local_dir = snapshot_download( + repo_id="lmsdss/regionreasoner_test_data", + repo_type="dataset", + allow_patterns=["raw/refcocog_multi_turn.json"], + cache_dir=Path(env["HF_HOME"]) / "hub" if "HF_HOME" in env else None, + ) + test_json = str(Path(local_dir) / "raw" / "refcocog_multi_turn.json") + + inference_script = ( + Path(rr_dir) / "test" / "evaluation" / "evaluation_multi_segmentation.py" + ) + if not inference_script.exists(): + raise FileNotFoundError( + f"RegionReasoner inference script not found: {inference_script}\n" + f"Check that REGION_REASONER_DIR={rr_dir!r} points to a valid clone." + ) + + with tempfile.TemporaryDirectory(prefix="rr_shards_") as tmp_dir: + procs = [] + for idx in range(num_gpus): + shard_env = dict(env) + shard_env["CUDA_VISIBLE_DEVICES"] = str(idx) + cmd = [ + "python", + str(inference_script), + "--model_path", + model_path, + "--model", + model_type, + "--test_data_path", + test_json, + "--output_path", + tmp_dir, # script writes /output_{idx}.json + "--vis_output_path", + str(Path(tmp_dir) / f"vis_{idx}"), + "--idx", + str(idx), + "--num_parts", + str(num_parts), + "--batch_size", + "2", + "--task_router_model_path", + "Ricky06662/TaskRouter-1.5B", + ] + logger.info("Starting shard %d/%d: %s", idx + 1, num_gpus, " ".join(cmd)) + proc = subprocess.Popen(cmd, env=shard_env, cwd=str(Path(test_json).parent)) + procs.append(proc) + + for idx, proc in enumerate(procs): + ret = proc.wait() + if ret != 0: + raise RuntimeError( + f"RegionReasoner inference shard {idx} exited with code {ret}" + ) + + logger.info("All %d shards completed. Computing metrics.", num_gpus) + + metrics = _aggregate_shards(tmp_dir) + + result_json = { + "model_name_or_path": model_path, + "results": {task: metrics}, + "configs": {task: {"num_fewshot": n_shot}}, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(result_json, f, indent=2) + logger.info("Results written to %s", output_path) + + +def _aggregate_shards(shard_dir: str) -> dict[str, float]: + """Read per-shard output files and compute all metrics via :mod:`metrics`. + + Each shard file contains a list of per-sample dicts with pre-computed + ``intersection``, ``union``, and ``bbox_iou`` fields written by the + upstream ``evaluation_multi_segmentation.py`` script. + + All metrics are computed through the :class:`BaseMetric` subclasses in + ``oellm.contrib.region_reasoner.metrics``. + + Returns a flat dict of ``{metric_name: value}`` for all seven metrics. + """ + from oellm.contrib.region_reasoner.metrics import ( + BboxAP, + CIoU, + GIoU, + PassRate, + ) + + shard_files = sorted(Path(shard_dir).glob("output_*.json")) + if not shard_files: + raise RuntimeError( + f"No shard output files found in {shard_dir!r}. " + "The inference script may have failed silently." + ) + + samples: list[str] = [] + for shard_file in shard_files: + with open(shard_file) as f: + shard_data = json.load(f) + for sample in shard_data: + samples.append(json.dumps(sample)) + + if not samples: + raise RuntimeError( + "No samples found across shard files. " + "The inference script produced empty output." + ) + logger.info("Aggregating %d samples from %d shards", len(samples), len(shard_files)) + + empty_refs = [""] * len(samples) + all_metrics = [ + GIoU(), + CIoU(), + BboxAP(), + PassRate(0.3), + PassRate(0.5), + PassRate(0.7), + PassRate(0.9), + ] + + metrics = {} + for m in all_metrics: + val = m.compute(samples, empty_refs) + metrics[m.name] = val + logger.debug("%s = %.4f", m.name, val) + return metrics + + +# --------------------------------------------------------------------------- +# Plugin protocol: required — parse results JSON +# --------------------------------------------------------------------------- + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Try to parse *data* as a region_reasoner output JSON. + + Returns ``(model_id, task_name, n_shot, {metric: value})`` if the JSON + matches this suite's format, otherwise ``None``. + + Detection heuristic: the ``results`` dict contains a key that starts with + ``"regionreasoner_"`` and the value dict contains ``"gIoU"``. + """ + results = data.get("results", {}) + for task_name, task_results in results.items(): + if task_name.startswith("regionreasoner_") and "gIoU" in task_results: + model_id = data.get("model_name_or_path") or data.get("model_name", "unknown") + n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) + return model_id, task_name, int(n_shot), task_results + return None diff --git a/oellm/contrib/region_reasoner/task.py b/oellm/contrib/region_reasoner/task.py new file mode 100644 index 00000000..55a74544 --- /dev/null +++ b/oellm/contrib/region_reasoner/task.py @@ -0,0 +1,50 @@ +"""RegionReasoner task definition.""" + +from oellm.core.base_task import BaseTask + + +class RegionReasonerTask(BaseTask): + """Multi-turn region grounding benchmark on RefCOCOg.""" + + @property + def name(self) -> str: + return "regionreasoner_refcocog" + + @property + def suite(self) -> str: + return "region_reasoner" + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def task_group_name(self) -> str: + return "region-reasoner" + + @property + def description(self) -> str: + return ( + "RegionReasoner multi-turn region grounding benchmark (RefCOCOg). " + "Requires REGION_REASONER_DIR on cluster." + ) + + @property + def primary_metric(self) -> str: + return "gIoU" + + @property + def hf_models(self) -> list[str]: + return ["Ricky06662/TaskRouter-1.5B", "facebook/sam2-hiera-large"] + + @property + def hf_dataset_files(self) -> list[dict]: + return [ + { + "repo_id": "lmsdss/regionreasoner_test_data", + "patterns": [ + "raw/refcocog_multi_turn.json", + "raw/refcocog_test_multi_bbox_images/*", + ], + } + ] diff --git a/oellm/core/base_metric.py b/oellm/core/base_metric.py index 3f63b618..8a5b5568 100644 --- a/oellm/core/base_metric.py +++ b/oellm/core/base_metric.py @@ -5,8 +5,8 @@ class BaseMetric(ABC): """Abstract base class for custom metric implementations. Implement this when an evaluation task requires a metric not natively - supported by lm-eval or lmms-eval (e.g. custom safety scores for T4.4, - or domain-specific metrics for T4.3). + supported by lm-eval or lmms-eval (e.g. a custom IoU score for grounding + benchmarks, or a domain-specific accuracy metric). The ``compute`` method must return a scalar in [0, 1] by convention, though higher-range metrics (e.g. OCRBench score /1000) are allowed when diff --git a/oellm/core/base_model_adapter.py b/oellm/core/base_model_adapter.py index 11f8a8cb..fb0615f9 100644 --- a/oellm/core/base_model_adapter.py +++ b/oellm/core/base_model_adapter.py @@ -5,14 +5,8 @@ class BaseModelAdapter(ABC): """Abstract base class for model adapters. - An adapter's sole responsibility is translating a model path/config into - the engine-specific argument strings passed on the command line. The - scheduling engine reads these strings and injects them into the sbatch - template — adapters never call the engines directly. - - This is the single integration point for adding new or proprietary models: - implement the two abstract methods and the adapter works with any engine - the platform supports. + Translates a model path/config into engine-specific argument strings + passed on the command line. Example:: @@ -50,3 +44,14 @@ def to_lmms_eval_args(self) -> str: Example: ``"pretrained=/path/to/model"`` """ + + def to_contrib_flags(self) -> str | None: + """Return the model-type flag for contrib suite routing. + + This is the value returned by ``detect_model_flags()`` in + ``suite.py``. Override to distinguish between inference backends + for the same benchmark (e.g. ``"vision_reasoner"`` vs ``"qwen2"``). + + Returns ``None`` by default (no model-type distinction needed). + """ + return None diff --git a/oellm/core/base_task.py b/oellm/core/base_task.py index df750f2d..f2ccd969 100644 --- a/oellm/core/base_task.py +++ b/oellm/core/base_task.py @@ -6,44 +6,50 @@ class BaseTask(ABC): """Abstract base class for evaluation task plugins. - Subclasses represent a single logical evaluation task and provide - the metadata needed for scheduling and dataset pre-download. + Example:: - This class is forward-looking: it is not yet consumed by the scheduling - engine. Teams building T4.2–T4.5 integrations should subclass BaseTask - to register new tasks without touching the YAML or core scheduling logic. - - Example (benchmark already in lmms-eval):: - - class VQAv2Task(BaseTask): + class MyTask(BaseTask): @property def name(self) -> str: - return "vqav2_val_all" + return "my_task" @property def suite(self) -> str: - return "lmms_eval" + return "my_suite" @property def n_shots(self) -> list[int]: return [0] @property - def dataset_specs(self) -> list[DatasetSpec]: - return [DatasetSpec(repo_id="HuggingFaceM4/VQAv2")] + def primary_metric(self) -> str | None: + return "my_metric" + + @property + def hf_models(self) -> list[str]: + return ["org/aux-model"] + + @property + def hf_dataset_files(self) -> list[dict]: + return [{"repo_id": "org/dataset", "patterns": ["data/*.json"]}] """ + # ------------------------------------------------------------------ + # Required + # ------------------------------------------------------------------ + @property @abstractmethod def name(self) -> str: - """Canonical task name, used as the CSV task_path column value.""" + """Canonical task name — used as the CSV ``task_path`` column value.""" @property @abstractmethod def suite(self) -> str: """Evaluation suite identifier. - Must be one of: ``lm_eval``, ``lighteval``, ``lmms_eval``. + One of: ``lm_eval``, ``lighteval``, ``lmms_eval``, or a contrib + ``SUITE_NAME`` (e.g. ``"region_reasoner"``). """ @property @@ -51,20 +57,106 @@ def suite(self) -> str: def n_shots(self) -> list[int]: """List of n-shot values to evaluate at.""" + # ------------------------------------------------------------------ + # Optional — override as needed + # ------------------------------------------------------------------ + @property def engine_task_name(self) -> str: """Task name as passed to the eval engine CLI. - Defaults to ``name``. Override when the engine's registered task name - differs from the canonical task name used in the CSV / YAML. + Defaults to :attr:`name`. Override when the engine's registered task + name differs from the canonical name used in the CSV / YAML. """ return self.name + @property + def description(self) -> str: + """Human-readable description shown in task group listings.""" + return "" + + @property + def task_group_name(self) -> str: + """Key used for this task in the ``task_groups`` dict. + + Defaults to :attr:`name` with underscores replaced by dashes. + """ + return self.name.replace("_", "-") + + @property + def primary_metric(self) -> str | None: + """Primary metric key written to the results CSV. + + Maps to the ``task_metrics`` entry in the generated TASK_GROUPS dict. + Return ``None`` to omit (collect_results will use its fallback chain). + """ + return None + @property def dataset_specs(self) -> list[DatasetSpec]: - """Dataset specifications for pre-download. + """HF datasets for pre-download via ``load_dataset()``. - Return an empty list if no pre-download is required (e.g. the dataset - is already available on the compute nodes or pre-downloaded elsewhere). + Use for small datasets accessed through the ``datasets`` library. + For large files downloaded via ``snapshot_download()``, use + :attr:`hf_dataset_files` instead. """ return [] + + @property + def hf_models(self) -> list[str]: + """Auxiliary HF model repos pre-downloaded via ``snapshot_download()``. + + List repos required at eval time but not the primary model under + evaluation (e.g. a task router or segmentation model). + """ + return [] + + @property + def hf_dataset_files(self) -> list[dict]: + """Specific dataset files pre-downloaded via ``snapshot_download()``. + + Use instead of :attr:`dataset_specs` when only a subset of a large + dataset repo is needed, or when files are read directly (not via the + ``datasets`` library). + + Each entry: ``{"repo_id": "org/dataset", "patterns": ["glob/..."]}`` + """ + return [] + + # ------------------------------------------------------------------ + # Factory + # ------------------------------------------------------------------ + + @classmethod + def to_task_groups_dict(cls) -> dict: + """Generate the ``TASK_GROUPS`` dict for use in ``suite.py``:: + + TASK_GROUPS: dict = MyTask.to_task_groups_dict() + + Note: + The subclass must be instantiable with no arguments. + """ + inst = cls() + + task_entry: dict = {"task": inst.name} + if inst.dataset_specs: + task_entry["dataset"] = inst.dataset_specs[0].repo_id + if inst.dataset_specs[0].subset: + task_entry["subset"] = inst.dataset_specs[0].subset + if inst.hf_models: + task_entry["hf_models"] = inst.hf_models + if inst.hf_dataset_files: + task_entry["hf_dataset_files"] = inst.hf_dataset_files + + task_group: dict = { + "suite": inst.suite, + "n_shots": inst.n_shots, + "tasks": [task_entry], + } + if inst.description: + task_group["description"] = inst.description + + result: dict = {"task_groups": {inst.task_group_name: task_group}} + if inst.primary_metric: + result["task_metrics"] = {inst.name: inst.primary_metric} + return result diff --git a/oellm/main.py b/oellm/main.py index b8c13bb7..797ab9a8 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -15,6 +15,8 @@ from oellm.task_groups import ( _collect_dataset_specs, + _collect_hf_dataset_files, + _collect_hf_model_repos, _expand_task_groups, _lookup_dataset_specs_for_tasks, ) @@ -25,6 +27,8 @@ _load_cluster_env, _num_jobs_in_queue, _pre_download_datasets_from_specs, + _pre_download_hf_dataset_files, + _pre_download_hf_model_repos, _process_model_paths, _setup_logging, capture_third_party_output_from_kwarg, @@ -134,15 +138,19 @@ def schedule_evals( else: logging.info("Skipping runtime environment check (--skip-checks enabled)") - if isinstance(models, str) and models is not None: + if isinstance(models, str): models = [m.strip() for m in models.split(",") if m.strip()] # type: ignore - if isinstance(tasks, str) and tasks is not None: + if isinstance(tasks, str): tasks = [t.strip() for t in tasks.split(",") if t.strip()] # type: ignore - if isinstance(n_shot, int) and n_shot is not None: + if isinstance(n_shot, int): n_shot = [n_shot] + group_names: list[str] | None = None + if task_groups: + group_names = [g.strip() for g in task_groups.split(",")] + eval_jobs: list[EvaluationJob] = [] if eval_csv_path: if models or tasks or task_groups or n_shot: @@ -161,8 +169,6 @@ def schedule_evals( else: df["eval_suite"] = df["eval_suite"].fillna("lm_eval") - # Always expand local model paths, even with skip_checks - df["model_path"].unique() eval_jobs.extend( [ EvaluationJob( @@ -176,7 +182,7 @@ def schedule_evals( ) elif models: - if task_groups is None: + if group_names is None: eval_jobs.extend( [ EvaluationJob( @@ -191,7 +197,7 @@ def schedule_evals( ] ) else: - expanded = _expand_task_groups([g.strip() for g in task_groups.split(",")]) + expanded = _expand_task_groups(group_names) eval_jobs.extend( [ EvaluationJob( @@ -223,11 +229,26 @@ def schedule_evals( # For lmms_eval jobs, encode the adapter class in eval_suite as "lmms_eval:". # This makes LMMS_MODEL_TYPE completely transparent — users never set it manually. + # For contrib suites, the registry's detect_model_flags() provides the same service. + from oellm import registry as _registry # noqa: PLC0415 + for job in expanded_eval_jobs: if job.eval_suite == "lmms_eval": adapter = _detect_lmms_model_type(str(job.model_path)) job.eval_suite = f"lmms_eval:{adapter}" logging.debug(f"lmms-eval adapter for {job.model_path}: {adapter}") + else: + try: + mod = _registry.get_suite(job.eval_suite) + if hasattr(mod, "detect_model_flags"): + flags = mod.detect_model_flags(str(job.model_path)) + if flags: + job.eval_suite = f"{job.eval_suite}:{flags}" + logging.debug( + f"Contrib suite flags for {job.model_path} ({mod.SUITE_NAME}): {flags}" + ) + except KeyError: + pass # Not a registered contrib suite — pass eval_suite through unchanged if not skip_checks: hub_models: set[str | Path] = { @@ -253,10 +274,8 @@ def schedule_evals( # network access on compute nodes. if not skip_checks: dataset_specs = [] - if task_groups: - dataset_specs = _collect_dataset_specs( - [g.strip() for g in task_groups.split(",")] - ) + if group_names: + dataset_specs = _collect_dataset_specs(group_names) else: # Look up individual tasks in task groups registry all_tasks = df["task_path"].unique().tolist() @@ -270,6 +289,18 @@ def schedule_evals( _pre_download_datasets_from_specs( dataset_specs, trust_remote_code=trust_remote_code ) + + hf_model_repos = [] + if group_names: + hf_model_repos = _collect_hf_model_repos(group_names) + if hf_model_repos: + _pre_download_hf_model_repos(hf_model_repos) + + hf_dataset_files = [] + if group_names: + hf_dataset_files = _collect_hf_dataset_files(group_names) + if hf_dataset_files: + _pre_download_hf_dataset_files(hf_dataset_files) else: logging.info("Skipping dataset pre-download (--skip-checks enabled)") @@ -309,7 +340,6 @@ def schedule_evals( sbatch_template = (files("oellm.resources") / "template.sbatch").read_text() - # Calculate dynamic array size and time limits total_evals = len(df) minutes_per_eval = 10 # Budget 10 minutes per eval total_minutes = total_evals * minutes_per_eval @@ -328,6 +358,11 @@ def schedule_evals( computed_time = f"{hours_with_margin:02d}:59:00" time_limit = computed_time + # clusters.yaml TIME_LIMIT overrides the computed value + if cluster_time_limit := os.environ.get("TIME_LIMIT"): + time_limit = cluster_time_limit + logging.info(f"Using TIME_LIMIT from clusters.yaml: {time_limit}") + # Apply slurm_template_var overrides (JSON object) if slurm_template_var: try: @@ -349,8 +384,7 @@ def schedule_evals( os.environ[key] = str(value) logging.info(f"Using slurm_template_var override: {key}={value}") - # Log the calculated values - logging.info("📊 Evaluation planning:") + logging.info("Evaluation planning:") logging.info(f" Total evaluations: {total_evals}") logging.info(f" Estimated time per eval: {minutes_per_eval} minutes") logging.info( @@ -401,12 +435,11 @@ def schedule_evals( try: logging.info("Calling sbatch to launch the evaluations") - # Provide helpful information about job monitoring and file locations - logging.info(f"📁 Evaluation directory: {evals_dir}") - logging.info(f"📄 SLURM script: {sbatch_script_path}") - logging.info(f"📋 Job configuration: {csv_path}") - logging.info(f"📜 SLURM logs will be stored in: {slurm_logs_dir}") - logging.info(f"📊 Results will be stored in: {evals_dir / 'results'}") + logging.info(f"Evaluation directory: {evals_dir}") + logging.info(f"SLURM script: {sbatch_script_path}") + logging.info(f"Job configuration: {csv_path}") + logging.info(f"SLURM logs: {slurm_logs_dir}") + logging.info(f"Results: {evals_dir / 'results'}") result = subprocess.run( ["sbatch"], @@ -421,9 +454,9 @@ def schedule_evals( job_id_match = re.search(r"Submitted batch job (\d+)", result.stdout) if job_id_match: job_id = job_id_match.group(1) - logging.info(f"🔍 Monitor job status: squeue -j {job_id}") - logging.info(f"📈 View job details: scontrol show job {job_id}") - logging.info(f"❌ Cancel job if needed: scancel {job_id}") + logging.info(f"Monitor job status: squeue -j {job_id}") + logging.info(f"View job details: scontrol show job {job_id}") + logging.info(f"Cancel job if needed: scancel {job_id}") except subprocess.CalledProcessError as e: logging.error(f"Failed to submit job: {e}") logging.error(f"sbatch stderr: {e.stderr}") @@ -460,6 +493,14 @@ def collect_results( _tg_cfg = yaml.safe_load(_f) task_metrics = _tg_cfg.get("task_metrics", {}) + # Merge contrib task_metrics so that custom benchmarks registered via the + # plugin registry are resolved correctly by _resolve_metric() below. + from oellm.registry import ( + get_all_task_groups as _contrib_task_groups, # noqa: PLC0415 + ) + + task_metrics.update(_contrib_task_groups().get("task_metrics", {})) + def _resolve_metric( task_name: str, result_dict: dict ) -> tuple[float | None, str | None]: @@ -471,8 +512,7 @@ def _resolve_metric( # below sees "vqa_score,none" regardless of engine. Keys without "/" # (lm-eval format) are passed through unchanged. result_dict = { - (k.split("/", 1)[1] if "/" in k else k): v - for k, v in result_dict.items() + (k.split("/", 1)[1] if "/" in k else k): v for k, v in result_dict.items() } # Skip non-metric keys; lm-eval uses suffixes like ",none" or ",remove_whitespace" @@ -511,7 +551,11 @@ def _first_matching_prefix( # Last resort: pick the first numeric non-stderr value (catches lmms-eval # benchmarks with non-standard metric names like mme_cognition_score) for k, v in result_dict.items(): - if isinstance(v, (int, float)) and "stderr" not in k and k not in ("alias", " ", ""): + if ( + isinstance(v, (int, float)) + and "stderr" not in k + and k not in ("alias", " ", "") + ): return float(v), k return None, None @@ -522,7 +566,11 @@ def _first_matching_prefix( # lm-eval writes flat JSON files: results/.json # lmms-eval writes nested dirs: results/.json//_results.json # rglob("*.json") + is_file() finds both without breaking backward compat. - search_root = (results_path / "results") if (results_path / "results").is_dir() else results_path + search_root = ( + (results_path / "results") + if (results_path / "results").is_dir() + else results_path + ) json_files = [p for p in search_root.rglob("*.json") if p.is_file()] if not json_files: @@ -542,20 +590,17 @@ def _first_matching_prefix( jobs_df = pd.read_csv(jobs_csv_path) logging.info(f"Found {len(jobs_df)} scheduled jobs in jobs.csv") - # Collect results rows = [] - completed_jobs = set() # Track (model, task, n_shot) tuples + completed_jobs = set() for json_file in json_files: with open(json_file) as f: data = json.load(f) - # Extract model name/path. # lmms-eval sets model_name to the adapter type (e.g. "llava_hf"), # not the checkpoint path; the actual path is in model_name_or_path. model_name = data.get("model_name_or_path") or data.get("model_name", "unknown") - # Extract results for each task results = data.get("results", {}) n_shot_data = data.get("n-shot", {}) @@ -634,7 +679,6 @@ def _first_matching_prefix( if task_name.startswith("global_mmlu_") and task_name.count("_") >= 4: continue - # Get n_shot for this task n_shot = n_shot_data.get(task_name, "unknown") # If this is a group aggregate and n_shot is missing, derive from any subtask @@ -669,11 +713,9 @@ def _first_matching_prefix( if set(task_results.keys()) <= {"alias", " ", ""}: continue - # Get the primary metric (usually acc, acc_norm) performance, metric_name = _resolve_metric(task_name, task_results) if performance is not None: - # Track completed job for check mode if check: completed_jobs.add((model_name, task_name, n_shot)) @@ -699,14 +741,12 @@ def _first_matching_prefix( logging.warning("No results extracted from JSON files") return - # Create DataFrame and save to CSV (if we have results) if rows: df = pd.DataFrame(rows) df.to_csv(output_csv, index=False) logging.info(f"Results saved to {output_csv}") logging.info(f"Extracted {len(df)} evaluation results") - # Print summary statistics if verbose: logging.info("Summary:") logging.info(f"Unique models: {df['model_name'].nunique()}") @@ -715,24 +755,19 @@ def _first_matching_prefix( f"N-shot values: {sorted(str(x) for x in df['n_shot'].unique())}" ) - # Perform check analysis if requested if check: logging.info("=== Evaluation Status Check ===") - # Find missing jobs missing_jobs = [] for _, job in jobs_df.iterrows(): job_tuple = (job["model_path"], job["task_path"], job["n_shot"]) - # Check if this job corresponds to one of our completed results is_completed = False - # Try exact matching first if job_tuple in completed_jobs: is_completed = True else: - # Try fuzzy matching for model names for completed_job in completed_jobs: completed_model, completed_task, completed_n_shot = completed_job @@ -765,7 +800,6 @@ def _first_matching_prefix( f"You can run these with: oellm schedule-eval --eval_csv_path {missing_csv}" ) - # Show some examples if verbose if verbose and len(missing_jobs) > 0: logging.info("Example missing jobs:") for _i, (_, job) in enumerate(missing_df.head(5).iterrows()): diff --git a/oellm/registry.py b/oellm/registry.py new file mode 100644 index 00000000..8c4c1ad7 --- /dev/null +++ b/oellm/registry.py @@ -0,0 +1,131 @@ +"""Contrib plugin registry. + +Auto-discovers suite modules under ``oellm/contrib/*/suite.py`` and provides a +unified view of all task groups and metric parsers contributed by them. + +Plugin protocol +--------------- +A file ``oellm/contrib//suite.py`` is a plugin if it exposes: + +Required +~~~~~~~~ +``SUITE_NAME: str`` + Identifier used in the ``eval_suite`` CSV column, e.g. ``"region_reasoner"``. + +``TASK_GROUPS: dict`` + Task-group definitions in ``task-groups.yaml`` format. Expected keys: + ``task_metrics``, ``task_groups``, ``super_groups`` (all optional). + +``run(*, model_path, task, n_shot, output_path, model_flags, env) -> None`` + Execute the evaluation. Must write a lmms-eval-compatible JSON file to + *output_path* so that :func:`oellm.main.collect_results` can parse it + without changes. + +``parse_results(data: dict) -> tuple | None`` + Try to parse a raw JSON dict produced by this suite. Returns + ``(model_id, task_name, n_shot, {metric: value})`` or ``None``. + +Optional +~~~~~~~~ +``CLUSTER_ENV_VARS: list[str]`` + Names of environment variables that must be set on the cluster. + Validated by ``oellm.contrib.dispatch`` before calling ``run()``. + +``detect_model_flags(model_path: str) -> str | None`` + Return a model-type suffix for the ``eval_suite`` column + (e.g. ``"vision_reasoner"``), or ``None``. + +Adding a new benchmark +---------------------- +Drop files into ``oellm/contrib//``. No core file changes required. +""" + +from __future__ import annotations + +import importlib +import logging +import pkgutil +import types +from functools import cache + +logger = logging.getLogger(__name__) + + +@cache +def _discover() -> dict[str, types.ModuleType]: + """Discover and return all suite modules, keyed by SUITE_NAME. + + Runs once per process (cached). Import errors in individual contribs are + logged as warnings and do not propagate. + """ + import oellm.contrib as _contrib_pkg + + suites: dict[str, types.ModuleType] = {} + + for _importer, modname, _ispkg in pkgutil.walk_packages( + path=_contrib_pkg.__path__, + prefix="oellm.contrib.", + onerror=lambda name: logger.warning("Error walking package %s", name), + ): + if not modname.endswith(".suite"): + continue + try: + mod = importlib.import_module(modname) + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to import contrib suite %s: %s", modname, exc) + continue + + if not hasattr(mod, "SUITE_NAME"): + continue + + suite_name: str = mod.SUITE_NAME + if suite_name in suites: + logger.warning( + "Duplicate SUITE_NAME %r: %s overrides %s", + suite_name, + modname, + suites[suite_name].__name__, + ) + suites[suite_name] = mod + logger.debug("Registered contrib suite: %r (%s)", suite_name, modname) + + return suites + + +def get_suite(name: str) -> types.ModuleType: + """Return the suite module registered under *name*. + + Raises + ------ + KeyError + If no suite with that name is registered. The error message includes + all known suite names to help diagnose typos. + """ + suites = _discover() + if name not in suites: + known = ", ".join(sorted(suites)) or "(none)" + raise KeyError( + f"Unknown contrib suite {name!r}. Known suites: {known}. " + "Make sure the suite module is under oellm/contrib//suite.py " + "and exposes a SUITE_NAME constant." + ) + return suites[name] + + +def get_all_suites() -> list[types.ModuleType]: + """Return all discovered suite modules.""" + return list(_discover().values()) + + +def get_all_task_groups() -> dict: + """Merge TASK_GROUPS from all discovered suites into a single dict. + + Returns a dict with keys ``task_metrics``, ``task_groups``, + ``super_groups`` suitable for merging into the core YAML data. + """ + merged: dict = {"task_metrics": {}, "task_groups": {}, "super_groups": {}} + for mod in _discover().values(): + tg = getattr(mod, "TASK_GROUPS", {}) + for key in ("task_metrics", "task_groups", "super_groups"): + merged[key].update(tg.get(key, {})) + return merged diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index beb5bc1b..7fbe5fd3 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -2,6 +2,7 @@ #SBATCH --job-name=oellm-eval #SBATCH --time={time_limit} #SBATCH --gres=gpu:$GPUS_PER_NODE +#SBATCH --mem=$MEM_PER_NODE #SBATCH --output={log_dir}/%x-%A-%a.out #SBATCH --partition=$PARTITION #SBATCH --account=$ACCOUNT @@ -101,6 +102,7 @@ do GPU_DEVICES=$(seq -s, 0 $(($GPUS_PER_NODE - 1))) + # Strip optional model-flags suffix ("region_reasoner:vision_reasoner" → "region_reasoner") suite_normalized=$(echo "${{eval_suite%%:*}}" | tr '[:upper:]' '[:lower:]') # Helper function to run Python commands in the appropriate environment @@ -198,7 +200,31 @@ do fi ;; *) - echo "[warning] Unknown evaluation suite '$eval_suite'. Skipping." + # Contrib suite: dispatch to the Python plugin registry. + # This single case handles ALL registered contrib benchmarks — no + # further template.sbatch changes are needed when adding new ones. + # The suite module's run() writes a lmms-eval-compatible JSON file. + _CONTRIB_OUTPUT="{evals_dir}/$(openssl rand -hex 5).json" + + if [ -n "$VENV_PATH" ]; then + source "$VENV_PATH/bin/activate" + python -m oellm.contrib.dispatch \ + --suite "$eval_suite" \ + --model_path "$model_path" \ + --task "$task_path" \ + --n_shot "$n_shot" \ + --output_path "$_CONTRIB_OUTPUT" + else + singularity exec $SINGULARITY_ARGS \ + --bind $BIND_PATHS \ + $EVAL_SIF_PATH \ + python -m oellm.contrib.dispatch \ + --suite "$eval_suite" \ + --model_path "$model_path" \ + --task "$task_path" \ + --n_shot "$n_shot" \ + --output_path "$_CONTRIB_OUTPUT" + fi ;; esac diff --git a/oellm/task_groups.py b/oellm/task_groups.py index bc4eea13..b51885f2 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -17,6 +17,8 @@ class _Task: n_shots: list[int] | None = None dataset: str | None = None subset: str | None = None + hf_models: list[str] | None = None + hf_dataset_files: list[dict] | None = None @dataclass @@ -47,12 +49,16 @@ def from_dict(cls, name: str, data: dict) -> "TaskGroup": task_n_shots = task_data.get("n_shots") task_dataset = task_data.get("dataset") task_subset = task_data.get("subset") + task_hf_models = task_data.get("hf_models") + task_hf_dataset_files = task_data.get("hf_dataset_files") tasks.append( _Task( name=task_name, n_shots=task_n_shots, dataset=task_dataset, subset=task_subset, + hf_models=task_hf_models, + hf_dataset_files=task_hf_dataset_files, ) ) @@ -109,6 +115,15 @@ def _parse_task_groups( yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) or {} ) + from oellm.registry import ( + get_all_task_groups as _contrib_task_groups, # noqa: PLC0415 + ) + + _contrib = _contrib_task_groups() + data.setdefault("task_metrics", {}).update(_contrib.get("task_metrics", {})) + data.setdefault("task_groups", {}).update(_contrib.get("task_groups", {})) + data.setdefault("super_groups", {}).update(_contrib.get("super_groups", {})) + task_groups: dict[str, TaskGroup] = {} for task_group_name, task_data in data["task_groups"].items(): @@ -135,6 +150,20 @@ class TaskGroupResult: suite: str +def _iter_all_tasks( + parsed: dict[str, TaskSuperGroup | TaskGroup], +) -> Iterable[tuple[_Task, str]]: + """Yield ``(task, suite)`` pairs from a parsed group dict, flattening super groups.""" + for group in parsed.values(): + if isinstance(group, TaskGroup): + for t in group.tasks: + yield t, group.suite + else: + for g in group.task_groups: + for t in g.tasks: + yield t, g.suite + + def _expand_task_groups(group_names: Iterable[str]) -> list[TaskGroupResult]: parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) missing = {str(n).strip() for n in group_names if str(n).strip()} - set(parsed.keys()) @@ -142,23 +171,9 @@ def _expand_task_groups(group_names: Iterable[str]) -> list[TaskGroupResult]: raise ValueError(f"Unknown task group(s): {', '.join(sorted(missing))}") results: list[TaskGroupResult] = [] - - for _, group in parsed.items(): - if isinstance(group, TaskGroup): - suite = group.suite - for t in group.tasks: - shots = [int(s) for s in (t.n_shots or [])] - for shot in shots: - results.append(TaskGroupResult(task=t.name, n_shot=shot, suite=suite)) - else: - for g in group.task_groups: - suite = g.suite - for t in g.tasks: - shots = [int(s) for s in (t.n_shots or [])] - for shot in shots: - results.append( - TaskGroupResult(task=t.name, n_shot=shot, suite=suite) - ) + for t, suite in _iter_all_tasks(parsed): + for shot in (int(s) for s in (t.n_shots or [])): + results.append(TaskGroupResult(task=t.name, n_shot=shot, suite=suite)) return results @@ -191,50 +206,68 @@ def add_spec(dataset: str | None, subset: str | None): seen.add(key) specs.append(DatasetSpec(repo_id=dataset, subset=subset)) - for _, group in parsed.items(): - if isinstance(group, TaskGroup): - for t in group.tasks: - if t.dataset == "facebook/flores" and not t.subset: - for lang in _extract_flores_subsets(t.name): - add_spec(t.dataset, lang) - else: - add_spec(t.dataset, t.subset) + for t, _ in _iter_all_tasks(parsed): + if t.dataset == "facebook/flores" and not t.subset: + for lang in _extract_flores_subsets(t.name): + add_spec(t.dataset, lang) else: - for g in group.task_groups: - for t in g.tasks: - if t.dataset == "facebook/flores" and not t.subset: - for lang in _extract_flores_subsets(t.name): - add_spec(t.dataset, lang) - else: - add_spec(t.dataset, t.subset) + add_spec(t.dataset, t.subset) return specs +def _collect_hf_model_repos(group_names: Iterable[str]) -> list[str]: + """Return deduplicated HF model repo IDs declared in task ``hf_models`` fields.""" + parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) + + repos: list[str] = [] + seen: set[str] = set() + + for t, _ in _iter_all_tasks(parsed): + for repo_id in t.hf_models or []: + if repo_id not in seen: + seen.add(repo_id) + repos.append(repo_id) + + return repos + + +def _collect_hf_dataset_files(group_names: Iterable[str]) -> list[dict]: + """Return deduplicated HF dataset file specs declared in task ``hf_dataset_files`` fields.""" + parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) + + file_specs: list[dict] = [] + seen: set[str] = set() + + for t, _ in _iter_all_tasks(parsed): + for spec in t.hf_dataset_files or []: + repo_id = spec.get("repo_id", "") + if repo_id and repo_id not in seen: + seen.add(repo_id) + file_specs.append(spec) + + return file_specs + + def _build_task_dataset_map() -> dict[str, list[DatasetSpec]]: - """Build a mapping from task names to their dataset specs from all task groups.""" - data = ( - yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) or {} - ) + """Build a mapping from task names to their dataset specs from all task groups. - all_group_names = list(data.get("task_groups", {}).keys()) + Includes both core YAML task groups and contrib task groups from the registry. + """ + all_group_names = get_all_task_group_names() parsed = _parse_task_groups(all_group_names) task_map: dict[str, list[DatasetSpec]] = {} - for _, group in parsed.items(): - if isinstance(group, TaskGroup): - for t in group.tasks: - if t.dataset and t.name not in task_map: - if t.dataset == "facebook/flores" and not t.subset: - task_map[t.name] = [ - DatasetSpec(repo_id=t.dataset, subset=lang) - for lang in _extract_flores_subsets(t.name) - ] - else: - task_map[t.name] = [ - DatasetSpec(repo_id=t.dataset, subset=t.subset) - ] + for t, _ in _iter_all_tasks(parsed): + if t.dataset and t.name not in task_map: + if t.dataset == "facebook/flores" and not t.subset: + task_map[t.name] = [ + DatasetSpec(repo_id=t.dataset, subset=lang) + for lang in _extract_flores_subsets(t.name) + ] + else: + task_map[t.name] = [DatasetSpec(repo_id=t.dataset, subset=t.subset)] return task_map @@ -261,8 +294,15 @@ def _lookup_dataset_specs_for_tasks(task_names: Iterable[str]) -> list[DatasetSp def get_all_task_group_names() -> list[str]: - """Return all available task group names (excluding super_groups).""" + """Return all available task group names (core + all contrib suites).""" data = ( yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) or {} ) - return list(data.get("task_groups", {}).keys()) + core_names = list(data.get("task_groups", {}).keys()) + + from oellm.registry import ( + get_all_task_groups as _contrib_task_groups, # noqa: PLC0415 + ) + + contrib_names = list(_contrib_task_groups().get("task_groups", {}).keys()) + return core_names + [n for n in contrib_names if n not in core_names] diff --git a/oellm/utils.py b/oellm/utils.py index faf1bd18..8b254c0e 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -57,7 +57,14 @@ def _ensure_singularity_image(image_name: str | None) -> None: "or use --exec_mode=venv with a virtual environment." ) - image_path = Path(os.getenv("EVAL_BASE_DIR")) / image_name + eval_base_dir = os.getenv("EVAL_BASE_DIR") + if not eval_base_dir: + raise RuntimeError( + "EVAL_BASE_DIR environment variable is not set. " + "It should be configured in clusters.yaml for this cluster." + ) + + image_path = Path(eval_base_dir) / image_name try: console = get_console() @@ -95,8 +102,7 @@ def _setup_logging(verbose: bool = False): class RichFormatter(logging.Formatter): def format(self, record): - record.msg = f"{record.getMessage()}" - return record.msg + return record.getMessage() rich_handler.setFormatter(RichFormatter()) @@ -299,6 +305,52 @@ def _process_model_paths(models: Iterable[str]): ) +def _pre_download_hf_model_repos(repo_ids: list[str]) -> None: + """Download auxiliary HF model repos (e.g. SAM2) required by contrib suites.""" + from huggingface_hub import snapshot_download + + console = get_console() + with console.status( + f"Downloading auxiliary models… {len(repo_ids)} repos", spinner="dots" + ) as status: + for idx, repo_id in enumerate(repo_ids, 1): + status.update(f"Downloading '{repo_id}' ({idx}/{len(repo_ids)})") + try: + snapshot_download( + repo_id=repo_id, + cache_dir=Path(os.getenv("HF_HOME")) / "hub" + if "HF_HOME" in os.environ + else None, + ) + except Exception as e: + logging.warning(f"Failed to download auxiliary model '{repo_id}': {e}") + + +def _pre_download_hf_dataset_files(dataset_files: list[dict]) -> None: + """Download specific files from HF dataset repos declared in task ``hf_dataset_files`` fields.""" + from huggingface_hub import snapshot_download + + console = get_console() + with console.status( + f"Downloading auxiliary dataset files… {len(dataset_files)} repos", spinner="dots" + ) as status: + for idx, spec in enumerate(dataset_files, 1): + repo_id = spec.get("repo_id", "") + patterns = spec.get("patterns") + status.update(f"Downloading '{repo_id}' ({idx}/{len(dataset_files)})") + try: + snapshot_download( + repo_id=repo_id, + repo_type="dataset", + allow_patterns=patterns, + cache_dir=Path(os.getenv("HF_HOME")) / "hub" + if "HF_HOME" in os.environ + else None, + ) + except Exception as e: + logging.warning(f"Failed to download dataset files from '{repo_id}': {e}") + + def _pre_download_datasets_from_specs( specs: Iterable, trust_remote_code: bool = True ) -> None: diff --git a/pyproject.toml b/pyproject.toml index 11053edb..46234a54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ ignore = [ "W191", # indentation contains tabs ] +[tool.ruff.lint.isort] +known-first-party = ["oellm"] + [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] diff --git a/tests/test_base_interfaces.py b/tests/test_base_interfaces.py index 84f8648d..480d930d 100644 --- a/tests/test_base_interfaces.py +++ b/tests/test_base_interfaces.py @@ -3,7 +3,6 @@ from oellm.core import BaseMetric, BaseModelAdapter, BaseTask from oellm.task_groups import DatasetSpec - # ── Concrete implementations for testing ───────────────────────────────────── @@ -41,7 +40,9 @@ def name(self) -> str: def compute(self, predictions: list[str], references: list[str]) -> float: if not predictions: return 0.0 - return sum(p == r for p, r in zip(predictions, references)) / len(predictions) + return sum(p == r for p, r in zip(predictions, references, strict=True)) / len( + predictions + ) class HFAdapter(BaseModelAdapter): diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 13f78552..58d736c2 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -8,7 +8,6 @@ from oellm.main import collect_results - # ── Helpers ─────────────────────────────────────────────────────────────────── @@ -183,11 +182,13 @@ def test_ocrbench_score_metric(self, tmp_path): df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(512.0) - def test_mathvista_acc_metric(self, tmp_path): + def test_mathvista_llm_judge_metric(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"mathvista_testmini": {"mathvista_testmini/acc,none": 0.49}}, + "results": { + "mathvista_testmini": {"mathvista_testmini/llm_as_judge_eval,none": 0.49} + }, "n-shot": {"mathvista_testmini": 0}, } df = run_collect(tmp_path, data) diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py index a44b01d4..eb8b50cb 100644 --- a/tests/test_image_task_groups.py +++ b/tests/test_image_task_groups.py @@ -44,16 +44,12 @@ def test_image_vqa_present_in_yaml(self): assert IMAGE_TASK_GROUP in all_groups def test_image_vqa_suite_is_lmms_eval(self): - data = yaml.safe_load( - (files("oellm.resources") / "task-groups.yaml").read_text() - ) + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) suite = data["task_groups"][IMAGE_TASK_GROUP]["suite"] assert suite == "lmms_eval" def test_image_vqa_has_eight_tasks(self): - data = yaml.safe_load( - (files("oellm.resources") / "task-groups.yaml").read_text() - ) + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) tasks = data["task_groups"][IMAGE_TASK_GROUP]["tasks"] assert len(tasks) == 8 @@ -112,6 +108,7 @@ def test_schedule_evals_dry_run_image_vqa(self, tmp_path): with ( patch("oellm.main._load_cluster_env"), patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.main._detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( @@ -136,6 +133,7 @@ def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): with ( patch("oellm.main._load_cluster_env"), patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.main._detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( @@ -149,5 +147,5 @@ def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): csv_files = list(tmp_path.glob("**/jobs.csv")) assert len(csv_files) == 1 df = pd.read_csv(csv_files[0]) - assert set(df["eval_suite"].unique()) == {"lmms_eval"} + assert all(s.startswith("lmms_eval") for s in df["eval_suite"].unique()) assert set(df["task_path"].unique()) == EXPECTED_TASKS diff --git a/tests/test_region_reasoner.py b/tests/test_region_reasoner.py new file mode 100644 index 00000000..fb129727 --- /dev/null +++ b/tests/test_region_reasoner.py @@ -0,0 +1,630 @@ +"""Tests for the RegionReasoner contrib benchmark integration.""" + +import json +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +from oellm.core.base_metric import BaseMetric +from oellm.core.base_task import BaseTask +from oellm.task_groups import ( + _collect_dataset_specs, + _collect_hf_dataset_files, + _expand_task_groups, + get_all_task_group_names, +) + +RR_TASK_GROUP = "region-reasoner" +RR_TASK_NAME = "regionreasoner_refcocog" +RR_TEST_DATA_REPO = "lmsdss/regionreasoner_test_data" + + +# --------------------------------------------------------------------------- +# Task group integration (end-to-end: registry → task_groups merge → YAML) +# --------------------------------------------------------------------------- + + +class TestRegionReasonerTaskGroup: + def test_task_group_present_in_all_names(self): + assert RR_TASK_GROUP in get_all_task_group_names() + + def test_task_group_expands_to_correct_task(self): + results = _expand_task_groups([RR_TASK_GROUP]) + task_names = {r.task for r in results} + assert RR_TASK_NAME in task_names + + def test_task_group_suite_is_region_reasoner(self): + results = _expand_task_groups([RR_TASK_GROUP]) + for r in results: + assert r.suite == "region_reasoner", ( + f"Expected suite 'region_reasoner', got '{r.suite}'" + ) + + def test_task_group_n_shot_is_zero(self): + results = _expand_task_groups([RR_TASK_GROUP]) + for r in results: + assert r.n_shot == 0 + + def test_no_dataset_pre_download(self): + # No HuggingFace dataset (load_dataset-style) is declared for this task group. + specs = _collect_dataset_specs([RR_TASK_GROUP]) + assert specs == [] + + def test_hf_dataset_files_declared(self): + import oellm.contrib.region_reasoner.suite as s + + tasks = s.TASK_GROUPS["task_groups"][RR_TASK_GROUP]["tasks"] + repo_ids = [ + spec["repo_id"] for task in tasks for spec in task.get("hf_dataset_files", []) + ] + assert RR_TEST_DATA_REPO in repo_ids + + def test_collect_hf_dataset_files_returns_correct_spec(self): + specs = _collect_hf_dataset_files([RR_TASK_GROUP]) + assert len(specs) == 1 + assert specs[0]["repo_id"] == RR_TEST_DATA_REPO + assert "raw/refcocog_multi_turn.json" in specs[0]["patterns"] + + def test_task_groups_generated_from_task_class(self): + # TASK_GROUPS must be generated from RegionReasonerTask, not hardcoded. + from oellm.contrib.region_reasoner.task import RegionReasonerTask + + generated = RegionReasonerTask.to_task_groups_dict() + import oellm.contrib.region_reasoner.suite as s + + assert s.TASK_GROUPS == generated + + +# --------------------------------------------------------------------------- +# BaseTask subclass +# --------------------------------------------------------------------------- + + +class TestRegionReasonerTask: + @pytest.fixture + def task(self): + from oellm.contrib.region_reasoner.task import RegionReasonerTask + + return RegionReasonerTask() + + def test_is_base_task_instance(self, task): + assert isinstance(task, BaseTask) + + def test_name(self, task): + assert task.name == RR_TASK_NAME + + def test_suite(self, task): + assert task.suite == "region_reasoner" + + def test_n_shots(self, task): + assert task.n_shots == [0] + + def test_dataset_specs_empty(self, task): + # Data is accessed via hf_dataset_files (snapshot_download), not load_dataset. + assert task.dataset_specs == [] + + def test_hf_dataset_files(self, task): + repo_ids = [f["repo_id"] for f in task.hf_dataset_files] + assert RR_TEST_DATA_REPO in repo_ids + + def test_hf_models(self, task): + assert "Ricky06662/TaskRouter-1.5B" in task.hf_models + assert "facebook/sam2-hiera-large" in task.hf_models + + def test_primary_metric(self, task): + assert task.primary_metric == "gIoU" + + def test_task_group_name(self, task): + assert task.task_group_name == RR_TASK_GROUP + + def test_engine_task_name_defaults_to_name(self, task): + assert task.engine_task_name == task.name + + def test_to_task_groups_dict_structure(self, task): + d = task.to_task_groups_dict() + assert "task_metrics" in d + assert d["task_metrics"][RR_TASK_NAME] == "gIoU" + assert RR_TASK_GROUP in d["task_groups"] + tasks = d["task_groups"][RR_TASK_GROUP]["tasks"] + assert any(t["task"] == RR_TASK_NAME for t in tasks) + + +# --------------------------------------------------------------------------- +# BaseMetric subclasses +# --------------------------------------------------------------------------- + + +def _sample(intersection: int, union: int, bbox_iou: float = 0.0) -> str: + """Helper: JSON-serialise a sample dict for metric inputs.""" + return json.dumps( + { + "intersection": intersection, + "union": union, + "bbox_iou": bbox_iou, + } + ) + + +class TestGIoU: + @pytest.fixture + def metric(self): + from oellm.contrib.region_reasoner.metrics import GIoU + + return GIoU() + + def test_is_base_metric(self, metric): + assert isinstance(metric, BaseMetric) + + def test_name(self, metric): + assert metric.name == "gIoU" + + def test_perfect_overlap(self, metric): + s = _sample(100, 100) + assert metric.compute([s], [""]) == pytest.approx(1.0) + + def test_zero_overlap(self, metric): + s = _sample(0, 200) + assert metric.compute([s], [""]) == pytest.approx(0.0) + + def test_partial_overlap(self, metric): + s = _sample(25, 175) + assert metric.compute([s], [""]) == pytest.approx(25 / 175, abs=1e-4) + + def test_mean_over_multiple_samples(self, metric): + perfect = _sample(100, 100) + zero = _sample(0, 200) + score = metric.compute([perfect, zero], ["", ""]) + assert score == pytest.approx(0.5) + + def test_empty_input(self, metric): + assert metric.compute([], []) == pytest.approx(0.0) + + def test_null_sample(self, metric): + score = metric.compute(["null"], [""]) + assert score == pytest.approx(0.0) + + +class TestCIoU: + @pytest.fixture + def metric(self): + from oellm.contrib.region_reasoner.metrics import CIoU + + return CIoU() + + def test_is_base_metric(self, metric): + assert isinstance(metric, BaseMetric) + + def test_name(self, metric): + assert metric.name == "cIoU" + + def test_perfect_overlap(self, metric): + s = _sample(100, 100) + assert metric.compute([s], [""]) == pytest.approx(1.0) + + def test_zero_overlap(self, metric): + s = _sample(0, 200) + assert metric.compute([s], [""]) == pytest.approx(0.0) + + def test_cumulative_formula_differs_from_giou(self, metric): + from oellm.contrib.region_reasoner.metrics import GIoU + + giou = GIoU() + # Sample 1: IoU = 100/100 = 1.0 + # Sample 2: IoU = 50/200 = 0.25 + s1 = _sample(100, 100) + s2 = _sample(50, 200) + preds = [s1, s2] + refs = ["", ""] + ciou_val = metric.compute(preds, refs) # (100+50)/(100+200) = 0.5 + giou_val = giou.compute(preds, refs) # (1.0+0.25)/2 = 0.625 + assert ciou_val == pytest.approx(0.5) + assert giou_val == pytest.approx(0.625) + assert ciou_val != pytest.approx(giou_val) + + def test_empty_input(self, metric): + assert metric.compute([], []) == pytest.approx(0.0) + + +class TestBboxAP: + @pytest.fixture + def metric(self): + from oellm.contrib.region_reasoner.metrics import BboxAP + + return BboxAP() + + def test_is_base_metric(self, metric): + assert isinstance(metric, BaseMetric) + + def test_name(self, metric): + assert metric.name == "bbox_AP" + + def test_all_correct(self, metric): + s = _sample(100, 100, bbox_iou=0.9) + assert metric.compute([s, s], ["", ""]) == pytest.approx(1.0) + + def test_none_correct(self, metric): + s = _sample(10, 200, bbox_iou=0.3) + assert metric.compute([s], [""]) == pytest.approx(0.0) + + def test_threshold_at_half(self, metric): + above = _sample(80, 100, bbox_iou=0.6) + below = _sample(10, 100, bbox_iou=0.4) + assert metric.compute([above, below], ["", ""]) == pytest.approx(0.5) + + def test_empty_input(self, metric): + assert metric.compute([], []) == pytest.approx(0.0) + + +class TestPassRate: + @pytest.fixture(params=[0.3, 0.5, 0.7, 0.9]) + def threshold(self, request): + return request.param + + def test_name_includes_threshold(self, threshold): + from oellm.contrib.region_reasoner.metrics import PassRate + + pr = PassRate(threshold) + assert pr.name == f"pass_rate_{threshold}" + + def test_is_base_metric(self): + from oellm.contrib.region_reasoner.metrics import PassRate + + assert isinstance(PassRate(0.5), BaseMetric) + + def test_all_pass(self): + from oellm.contrib.region_reasoner.metrics import PassRate + + s = _sample(100, 100) + pr = PassRate(0.5) + assert pr.compute([s, s], ["", ""]) == pytest.approx(1.0) + + def test_none_pass(self): + from oellm.contrib.region_reasoner.metrics import PassRate + + s = _sample(0, 100) + pr = PassRate(0.3) + assert pr.compute([s], [""]) == pytest.approx(0.0) + + def test_half_pass(self): + from oellm.contrib.region_reasoner.metrics import PassRate + + perfect = _sample(100, 100) + zero = _sample(0, 100) + pr = PassRate(0.5) + score = pr.compute([perfect, zero], ["", ""]) + assert score == pytest.approx(0.5) + + def test_invalid_threshold_raises(self): + from oellm.contrib.region_reasoner.metrics import PassRate + + with pytest.raises(ValueError): + PassRate(0.0) + with pytest.raises(ValueError): + PassRate(1.0) + with pytest.raises(ValueError): + PassRate(-0.1) + + def test_empty_input(self): + from oellm.contrib.region_reasoner.metrics import PassRate + + assert PassRate(0.5).compute([], []) == pytest.approx(0.0) + + +# --------------------------------------------------------------------------- +# Suite plugin protocol +# --------------------------------------------------------------------------- + + +class TestSuiteProtocol: + @pytest.fixture + def suite(self): + import oellm.contrib.region_reasoner.suite as s + + return s + + def test_suite_name(self, suite): + assert suite.SUITE_NAME == "region_reasoner" + + def test_cluster_env_vars_declared(self, suite): + assert "REGION_REASONER_DIR" in suite.CLUSTER_ENV_VARS + + def test_task_groups_structure(self, suite): + tg = suite.TASK_GROUPS + assert "task_metrics" in tg + assert "task_groups" in tg + assert RR_TASK_NAME in tg["task_metrics"] + assert RR_TASK_GROUP in tg["task_groups"] + + def test_detect_model_flags_region_reasoner_model(self, suite): + assert suite.detect_model_flags("lmsdss/RegionReasoner-7B") == "vision_reasoner" + + def test_detect_model_flags_qwen2_model(self, suite): + assert suite.detect_model_flags("Qwen/Qwen2.5-VL-7B-Instruct") == "qwen2" + + def test_detect_model_flags_qwen1_model(self, suite): + assert suite.detect_model_flags("Qwen/Qwen-VL-Chat") == "qwen" + + def test_detect_model_flags_unknown_defaults_to_vision_reasoner(self, suite): + assert suite.detect_model_flags("some/unknown-model") == "vision_reasoner" + + def test_parse_results_valid_json(self, suite): + data = { + "model_name_or_path": "/path/to/model", + "results": { + RR_TASK_NAME: { + "gIoU": 0.42, + "cIoU": 0.45, + "bbox_AP": 0.38, + } + }, + "configs": {RR_TASK_NAME: {"num_fewshot": 0}}, + } + result = suite.parse_results(data) + assert result is not None + model_id, task_name, n_shot, metrics = result + assert model_id == "/path/to/model" + assert task_name == RR_TASK_NAME + assert n_shot == 0 + assert metrics["gIoU"] == pytest.approx(0.42) + + def test_parse_results_non_matching_json_returns_none(self, suite): + # lm-eval output format — should not be parsed by this suite + data = { + "model_name": "some_model", + "results": {"mmlu": {"acc,none": 0.55}}, + "n-shot": {"mmlu": 5}, + } + assert suite.parse_results(data) is None + + def test_parse_results_empty_results_returns_none(self, suite): + assert suite.parse_results({}) is None + + +# --------------------------------------------------------------------------- +# ModelAdapter +# --------------------------------------------------------------------------- + + +class TestRegionReasonerModelAdapter: + @pytest.fixture + def adapter_cls(self): + from oellm.contrib.region_reasoner.adapter import RegionReasonerModelAdapter + from oellm.core.base_model_adapter import BaseModelAdapter + + return RegionReasonerModelAdapter, BaseModelAdapter + + def test_is_base_model_adapter(self, adapter_cls): + cls, base = adapter_cls + assert issubclass(cls, base) + + def test_contrib_flags_region_reasoner(self, adapter_cls): + cls, _ = adapter_cls + assert cls("lmsdss/RegionReasoner-7B").to_contrib_flags() == "vision_reasoner" + + def test_contrib_flags_qwen2(self, adapter_cls): + cls, _ = adapter_cls + assert cls("Qwen/Qwen2.5-VL-7B").to_contrib_flags() == "qwen2" + + def test_contrib_flags_qwen(self, adapter_cls): + cls, _ = adapter_cls + assert cls("Qwen/Qwen-VL-Chat").to_contrib_flags() == "qwen" + + def test_contrib_flags_unknown_defaults_to_vision_reasoner(self, adapter_cls): + cls, _ = adapter_cls + assert cls("some/unknown-model").to_contrib_flags() == "vision_reasoner" + + def test_detect_model_flags_delegates_to_adapter(self): + import oellm.contrib.region_reasoner.suite as s + + assert s.detect_model_flags("lmsdss/RegionReasoner-7B") == "vision_reasoner" + assert s.detect_model_flags("Qwen/Qwen2.5-VL-7B") == "qwen2" + + +# --------------------------------------------------------------------------- +# Schedule evals dry-run integration +# --------------------------------------------------------------------------- + + +class TestRegionReasonerSchedule: + def test_schedule_evals_dry_run(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.main._load_cluster_env"), + patch("oellm.main._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="lmsdss/RegionReasoner-7B", + task_groups=RR_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + sbatch_content = sbatch_files[0].read_text() + # The contrib catch-all case must be present + assert "oellm.contrib.dispatch" in sbatch_content + + def test_jobs_csv_has_region_reasoner_suite(self, tmp_path): + import pandas as pd + + from oellm.main import schedule_evals + + with ( + patch("oellm.main._load_cluster_env"), + patch("oellm.main._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="lmsdss/RegionReasoner-7B", + task_groups=RR_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + csv_files = list(tmp_path.glob("**/jobs.csv")) + assert len(csv_files) == 1 + df = pd.read_csv(csv_files[0]) + # eval_suite should start with "region_reasoner" + assert all(s.startswith("region_reasoner") for s in df["eval_suite"]) + assert set(df["task_path"]) == {RR_TASK_NAME} + + +# --------------------------------------------------------------------------- +# _aggregate_shards (actual metric computation path) +# --------------------------------------------------------------------------- + + +class TestAggregateShards: + """Verify _aggregate_shards routes through metrics.py BaseMetric classes.""" + + def _write_shard(self, shard_dir, idx, samples): + path = shard_dir / f"output_{idx}.json" + path.write_text(json.dumps(samples)) + + def test_perfect_overlap(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [{"intersection": 100, "union": 100, "bbox_iou": 1.0}], + ) + m = _aggregate_shards(str(tmp_path)) + assert m["gIoU"] == pytest.approx(1.0) + assert m["cIoU"] == pytest.approx(1.0) + assert m["bbox_AP"] == pytest.approx(1.0) + assert m["pass_rate_0.3"] == pytest.approx(1.0) + assert m["pass_rate_0.9"] == pytest.approx(1.0) + + def test_zero_overlap(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [{"intersection": 0, "union": 200, "bbox_iou": 0.0}], + ) + m = _aggregate_shards(str(tmp_path)) + assert m["gIoU"] == pytest.approx(0.0) + assert m["cIoU"] == pytest.approx(0.0) + assert m["bbox_AP"] == pytest.approx(0.0) + assert m["pass_rate_0.3"] == pytest.approx(0.0) + + def test_pass_rates_differ_across_thresholds(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [ + {"intersection": 100, "union": 100, "bbox_iou": 1.0}, + {"intersection": 10, "union": 200, "bbox_iou": 0.05}, + ], + ) + m = _aggregate_shards(str(tmp_path)) + # mask IoU=1.0 passes all; mask IoU=0.05 passes none + assert m["pass_rate_0.3"] == pytest.approx(0.5) + assert m["pass_rate_0.5"] == pytest.approx(0.5) + assert m["pass_rate_0.7"] == pytest.approx(0.5) + assert m["pass_rate_0.9"] == pytest.approx(0.5) + + def test_pass_rates_actually_differ(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [ + {"intersection": 100, "union": 100, "bbox_iou": 1.0}, + {"intersection": 64, "union": 100, "bbox_iou": 0.64}, + ], + ) + m = _aggregate_shards(str(tmp_path)) + # mask IoU=1.0 passes all; mask IoU=0.64 passes 0.3 and 0.5 but not 0.7, 0.9 + assert m["pass_rate_0.3"] == pytest.approx(1.0) + assert m["pass_rate_0.5"] == pytest.approx(1.0) + assert m["pass_rate_0.7"] == pytest.approx(0.5) + assert m["pass_rate_0.9"] == pytest.approx(0.5) + + def test_multiple_shards_aggregated(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [{"intersection": 100, "union": 100, "bbox_iou": 1.0}], + ) + self._write_shard( + tmp_path, + 1, + [{"intersection": 0, "union": 100, "bbox_iou": 0.0}], + ) + m = _aggregate_shards(str(tmp_path)) + assert m["gIoU"] == pytest.approx(0.5) + assert m["cIoU"] == pytest.approx(0.5) + + def test_no_shard_files_raises(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + with pytest.raises(RuntimeError, match="No shard output files"): + _aggregate_shards(str(tmp_path)) + + def test_empty_shard_raises(self, tmp_path): + from oellm.contrib.region_reasoner.suite import _aggregate_shards + + self._write_shard(tmp_path, 0, []) + with pytest.raises(RuntimeError, match="No samples found"): + _aggregate_shards(str(tmp_path)) + + +# --------------------------------------------------------------------------- +# collect_results compatibility +# --------------------------------------------------------------------------- + + +class TestCollectResultsCompatibility: + """Verify collect_results() parses RegionReasoner output without modification.""" + + def test_collect_results_parses_region_reasoner_json(self, tmp_path): + import pandas as pd + + from oellm.main import collect_results + + results_dir = tmp_path / "results" + results_dir.mkdir() + + # Write a mock RegionReasoner output JSON (lmms-eval-compatible format) + mock_output = { + "model_name_or_path": "/cluster/models/RegionReasoner-7B", + "results": { + RR_TASK_NAME: { + "gIoU": 0.42, + "cIoU": 0.45, + "bbox_AP": 0.38, + "pass_rate_0.3": 0.71, + "pass_rate_0.5": 0.55, + } + }, + "configs": {RR_TASK_NAME: {"num_fewshot": 0}}, + } + (results_dir / "abc123.json").write_text(json.dumps(mock_output)) + + output_csv = str(tmp_path / "results.csv") + collect_results(str(tmp_path), output_csv=output_csv) + + assert Path(output_csv).exists() + df = pd.read_csv(output_csv) + assert len(df) == 1 + row = df.iloc[0] + assert row["task"] == RR_TASK_NAME + assert row["metric_name"] in ("gIoU", "gIoU,none") # primary metric + assert float(row["performance"]) == pytest.approx(0.42) + assert row["model_name"] == "/cluster/models/RegionReasoner-7B" diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 00000000..283a220d --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,62 @@ +"""Tests for the contrib plugin registry (oellm/registry.py).""" + +import pytest + +from oellm import registry + + +class TestRegistryDiscovery: + def test_region_reasoner_is_discovered(self): + """The RegionReasoner suite must be auto-discovered from contrib/.""" + suites = registry.get_all_suites() + suite_names = [getattr(mod, "SUITE_NAME", None) for mod in suites] + assert "region_reasoner" in suite_names + + def test_get_suite_returns_module(self): + mod = registry.get_suite("region_reasoner") + assert mod is not None + assert hasattr(mod, "SUITE_NAME") + assert mod.SUITE_NAME == "region_reasoner" + + def test_get_suite_unknown_raises_keyerror(self): + with pytest.raises(KeyError, match="nonexistent_suite_xyz"): + registry.get_suite("nonexistent_suite_xyz") + + def test_keyerror_message_lists_known_suites(self): + with pytest.raises(KeyError) as exc_info: + registry.get_suite("nonexistent_suite_xyz") + # The error message should mention known suites to help diagnose typos + assert "region_reasoner" in str(exc_info.value) + + def test_get_all_suites_returns_list(self): + suites = registry.get_all_suites() + assert isinstance(suites, list) + assert len(suites) >= 1 + + def test_suite_has_required_protocol_attributes(self): + mod = registry.get_suite("region_reasoner") + assert hasattr(mod, "SUITE_NAME"), "suite.py must expose SUITE_NAME" + assert hasattr(mod, "TASK_GROUPS"), "suite.py must expose TASK_GROUPS" + assert callable(getattr(mod, "run", None)), "suite.py must expose run()" + assert callable(getattr(mod, "parse_results", None)), ( + "suite.py must expose parse_results()" + ) + + +class TestRegistryTaskGroupMerge: + def test_task_metrics_contains_region_reasoner(self): + merged = registry.get_all_task_groups() + assert "regionreasoner_refcocog" in merged.get("task_metrics", {}) + + def test_task_groups_contains_region_reasoner(self): + merged = registry.get_all_task_groups() + assert "region-reasoner" in merged.get("task_groups", {}) + + def test_merged_task_group_has_correct_suite(self): + merged = registry.get_all_task_groups() + tg = merged["task_groups"]["region-reasoner"] + assert tg["suite"] == "region_reasoner" + + def test_merged_primary_metric(self): + merged = registry.get_all_task_groups() + assert merged["task_metrics"]["regionreasoner_refcocog"] == "gIoU" diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index ad6152bb..39a2cb90 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -19,6 +19,7 @@ def test_schedule_evals(tmp_path, n_shot, task_groups): with ( patch("oellm.main._load_cluster_env"), patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.main._detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( From ffeef35ccbf5d34cf2b99db3909558eede3d51ce Mon Sep 17 00:00:00 2001 From: islobozhan Date: Tue, 24 Mar 2026 13:11:58 +0100 Subject: [PATCH 09/44] [Base][Refactoring] Config validation and decompose main.py 1. Fix HF_HOME crash when env var is unset 2. Validate required cluster variables after loading clusters.yaml 3. Replace hardcoded lmms-eval adapter detection with LMMS_MODEL_ADAPTERS lookup 4. Remove magic-number time budgeting to use TIME_LIMIT from clusters.yaml directly 5. Decompose main.py into constants.py and results.py --- oellm/constants.py | 56 ++++++ oellm/main.py | 419 +-------------------------------------------- oellm/results.py | 381 +++++++++++++++++++++++++++++++++++++++++ oellm/utils.py | 21 ++- 4 files changed, 466 insertions(+), 411 deletions(-) create mode 100644 oellm/constants.py create mode 100644 oellm/results.py diff --git a/oellm/constants.py b/oellm/constants.py new file mode 100644 index 00000000..15915a5d --- /dev/null +++ b/oellm/constants.py @@ -0,0 +1,56 @@ +"""Shared constants and data structures used across the oellm package.""" + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class EvaluationJob: + model_path: Path | str + task_path: str + n_shot: int + eval_suite: str + + +# Mapping of model path patterns to lmms-eval adapter class names. +# Patterns are matched case-insensitively against the model path. +# Order matters: more specific patterns must come before general ones. +LMMS_MODEL_ADAPTERS: list[tuple[list[str], str]] = [ + (["qwen2.5-vl", "qwen2_5_vl", "qwen2.5vl"], "qwen2_5_vl"), + (["qwen2-vl", "qwen2_vl"], "qwen2_vl"), + (["llava"], "llava_hf"), + (["internvl"], "internvl2"), + (["idefics"], "idefics3"), + (["minicpm"], "minicpm_v"), + (["qwen"], "qwen_vl"), +] + +# Fallback metric keys used by collect_results when no task_metrics entry +# is found. Tried in order; first match wins. +METRIC_FALLBACK_KEYS: list[str] = [ + "acc,none", + "acc", + "accuracy", + "f1", + "exact_match", +] + + +def detect_lmms_model_type(model_path: str) -> str: + """Detect the lmms-eval adapter class name from a model path or HF repo name. + + lmms-eval requires --model (e.g. llava_hf, qwen2_5_vl). + This is inferred from the model name so users never need to set it manually. + + To add support for a new model family, add an entry to LMMS_MODEL_ADAPTERS + above or register a BaseModelAdapter via the contrib plugin system. + """ + name = str(model_path).lower() + for patterns, adapter in LMMS_MODEL_ADAPTERS: + if any(p in name for p in patterns): + return adapter + raise ValueError( + f"Cannot auto-detect lmms-eval adapter class from model path '{model_path}'. " + "Add a pattern to LMMS_MODEL_ADAPTERS in constants.py or register a " + "BaseModelAdapter via a contrib plugin." + ) diff --git a/oellm/main.py b/oellm/main.py index 797ab9a8..9f02da02 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -4,7 +4,6 @@ import os import re import subprocess -from dataclasses import dataclass from datetime import datetime from importlib.resources import files from pathlib import Path @@ -13,6 +12,8 @@ import pandas as pd from jsonargparse import auto_cli +from oellm.constants import EvaluationJob, detect_lmms_model_type +from oellm.results import collect_results from oellm.task_groups import ( _collect_dataset_specs, _collect_hf_dataset_files, @@ -34,41 +35,8 @@ capture_third_party_output_from_kwarg, ) - -@dataclass -class EvaluationJob: - model_path: Path | str - task_path: str - n_shot: int - eval_suite: str - - -def _detect_lmms_model_type(model_path: str) -> str: - """Detect the lmms-eval adapter class name from a model path or HF repo name. - - lmms-eval requires --model (e.g. llava_hf, qwen2_5_vl). - This is inferred from the model name so users never need to set it manually. - To add support for a new model family, add a pattern here. - """ - name = str(model_path).lower() - if "qwen2.5-vl" in name or "qwen2_5_vl" in name or "qwen2.5vl" in name: - return "qwen2_5_vl" - if "qwen2-vl" in name or "qwen2_vl" in name: - return "qwen2_vl" - if "llava" in name: - return "llava_hf" - if "internvl" in name: - return "internvl2" - if "idefics" in name: - return "idefics3" - if "minicpm" in name: - return "minicpm_v" - if "qwen" in name: - return "qwen_vl" - raise ValueError( - f"Cannot auto-detect lmms-eval adapter class from model path '{model_path}'. " - "Add your model family to _detect_lmms_model_type() in main.py." - ) +# Backward-compatible alias +_detect_lmms_model_type = detect_lmms_model_type @capture_third_party_output_from_kwarg("verbose") @@ -341,27 +309,10 @@ def schedule_evals( sbatch_template = (files("oellm.resources") / "template.sbatch").read_text() total_evals = len(df) - minutes_per_eval = 10 # Budget 10 minutes per eval - total_minutes = total_evals * minutes_per_eval - max_minutes_per_job = 18 * 60 # 18 hours - min_array_size_for_time = max(1, int(math.ceil(total_minutes / max_minutes_per_job))) - desired_array_size = min(128, total_evals) if total_evals >= 128 else total_evals - if desired_array_size < min_array_size_for_time: - desired_array_size = min_array_size_for_time - actual_array_size = min(remaining_queue_capacity, desired_array_size, total_evals) + actual_array_size = min(remaining_queue_capacity, total_evals) evals_per_job = max(1, int(math.ceil(total_evals / actual_array_size))) - minutes_per_job = evals_per_job * minutes_per_eval - minutes_with_margin = int(minutes_per_job * 1.2) - hours_with_margin = max(1, int(math.ceil(minutes_with_margin / 60))) - hours_with_margin = max(hours_with_margin, 3) - hours_with_margin = min(hours_with_margin, 23) - computed_time = f"{hours_with_margin:02d}:59:00" - time_limit = computed_time - - # clusters.yaml TIME_LIMIT overrides the computed value - if cluster_time_limit := os.environ.get("TIME_LIMIT"): - time_limit = cluster_time_limit - logging.info(f"Using TIME_LIMIT from clusters.yaml: {time_limit}") + + time_limit = os.environ.get("TIME_LIMIT", "12:00:00") # Apply slurm_template_var overrides (JSON object) if slurm_template_var: @@ -386,19 +337,11 @@ def schedule_evals( logging.info("Evaluation planning:") logging.info(f" Total evaluations: {total_evals}") - logging.info(f" Estimated time per eval: {minutes_per_eval} minutes") - logging.info( - f" Total estimated time: {total_minutes} minutes ({total_minutes / 60:.1f} hours)" - ) - logging.info(f" Desired array size: {desired_array_size}") logging.info( - f" Actual array size: {actual_array_size} (limited by queue capacity: {remaining_queue_capacity})" + f" Array size: {actual_array_size} (queue capacity: {remaining_queue_capacity})" ) logging.info(f" Evaluations per job: {evals_per_job}") - logging.info( - f" Time per job: {minutes_per_job} minutes ({minutes_per_job / 60:.1f} hours)" - ) - logging.info(f" Time limit with safety margin: {time_limit}") + logging.info(f" Time limit: {time_limit}") sbatch_script = sbatch_template.format( csv_path=csv_path, @@ -466,350 +409,6 @@ def schedule_evals( ) -def collect_results( - results_dir: str, - output_csv: str = "eval_results.csv", - *, - check: bool = False, - verbose: bool = False, -) -> None: - """ - Collect evaluation results from JSON files and export to CSV. - - Args: - results_dir: Path to the directory containing result JSON files - output_csv: Output CSV filename (default: eval_results.csv) - check: Check for missing evaluations and create a missing jobs CSV - verbose: Enable verbose logging - """ - import json - - import yaml - - _setup_logging(verbose) - - task_groups_yaml = files("oellm.resources") / "task-groups.yaml" - with open(str(task_groups_yaml)) as _f: - _tg_cfg = yaml.safe_load(_f) - task_metrics = _tg_cfg.get("task_metrics", {}) - - # Merge contrib task_metrics so that custom benchmarks registered via the - # plugin registry are resolved correctly by _resolve_metric() below. - from oellm.registry import ( - get_all_task_groups as _contrib_task_groups, # noqa: PLC0415 - ) - - task_metrics.update(_contrib_task_groups().get("task_metrics", {})) - - def _resolve_metric( - task_name: str, result_dict: dict - ) -> tuple[float | None, str | None]: - """Return (value, metric_name) for task_name from result_dict.""" - - # Normalise lmms-eval task-scoped metric keys so lm-eval and lmms-eval - # output is handled identically. lmms-eval writes keys like - # "vqav2/vqa_score,none"; strip the "task_name/" prefix so the lookup - # below sees "vqa_score,none" regardless of engine. Keys without "/" - # (lm-eval format) are passed through unchanged. - result_dict = { - (k.split("/", 1)[1] if "/" in k else k): v for k, v in result_dict.items() - } - - # Skip non-metric keys; lm-eval uses suffixes like ",none" or ",remove_whitespace" - def _first_numeric(d: dict, *candidates: str) -> tuple[float | None, str | None]: - for c in candidates: - if c in d and isinstance(d[c], (int, float)): - return float(d[c]), c - return None, None - - def _first_matching_prefix( - d: dict, prefix: str - ) -> tuple[float | None, str | None]: - for k, v in d.items(): - if (k == prefix or k.startswith(prefix + ",")) and isinstance( - v, (int, float) - ): - return float(v), k - return None, None - - preferred = task_metrics.get(task_name) - if preferred is not None: - val, key = _first_numeric(result_dict, f"{preferred},none", preferred) - if val is not None: - return val, key - val, key = _first_matching_prefix(result_dict, preferred) - return val, key - - for metric in ["acc,none", "acc", "accuracy", "f1", "exact_match"]: - val, key = _first_numeric(result_dict, metric) - if val is not None: - return val, key - val, key = _first_matching_prefix(result_dict, metric.split(",")[0]) - if val is not None: - return val, key - - # Last resort: pick the first numeric non-stderr value (catches lmms-eval - # benchmarks with non-standard metric names like mme_cognition_score) - for k, v in result_dict.items(): - if ( - isinstance(v, (int, float)) - and "stderr" not in k - and k not in ("alias", " ", "") - ): - return float(v), k - return None, None - - results_path = Path(results_dir) - if not results_path.exists(): - raise ValueError(f"Results directory does not exist: {results_dir}") - - # lm-eval writes flat JSON files: results/.json - # lmms-eval writes nested dirs: results/.json//_results.json - # rglob("*.json") + is_file() finds both without breaking backward compat. - search_root = ( - (results_path / "results") - if (results_path / "results").is_dir() - else results_path - ) - json_files = [p for p in search_root.rglob("*.json") if p.is_file()] - - if not json_files: - logging.warning(f"No JSON files found in {results_dir}") - if not check: - return - - logging.info(f"Found {len(json_files)} result files") - - # If check mode, also load the jobs.csv to compare - if check: - jobs_csv_path = results_path / "jobs.csv" - if not jobs_csv_path.exists(): - logging.warning(f"No jobs.csv found in {results_dir}, cannot perform check") - check = False - else: - jobs_df = pd.read_csv(jobs_csv_path) - logging.info(f"Found {len(jobs_df)} scheduled jobs in jobs.csv") - - rows = [] - completed_jobs = set() - - for json_file in json_files: - with open(json_file) as f: - data = json.load(f) - - # lmms-eval sets model_name to the adapter type (e.g. "llava_hf"), - # not the checkpoint path; the actual path is in model_name_or_path. - model_name = data.get("model_name_or_path") or data.get("model_name", "unknown") - - results = data.get("results", {}) - n_shot_data = data.get("n-shot", {}) - - # lmms-eval has no "n-shot" dict; fall back to per-task config "num_fewshot" - if not n_shot_data: - for _task, _cfg in data.get("configs", {}).items(): - if isinstance(_cfg, dict): - shot = _cfg.get("num_fewshot") - if shot is not None: - n_shot_data[_task] = shot - - # Infer a global n_shot if exactly one unique value exists in this JSON - global_n_shot = None - try: - candidate_values = [] - for _v in n_shot_data.values(): - if isinstance(_v, (int | float)): - candidate_values.append(int(_v)) - elif isinstance(_v, str) and _v.isdigit(): - candidate_values.append(int(_v)) - unique_values = set(candidate_values) - if len(unique_values) == 1: - global_n_shot = next(iter(unique_values)) - except Exception: - pass - - # Aggregate groups (lm-eval harness) - groups_map = data.get("groups", {}) - group_subtasks_map = data.get("group_subtasks", {}) - group_aggregate_names = set(groups_map.keys()) | set(group_subtasks_map.keys()) - group_subtask_names: set[str] = set() - for _agg, _subs in group_subtasks_map.items(): - for _s in _subs: - group_subtask_names.add(_s) - - # Prefer only the first aggregate metric from groups (simplified) - if groups_map: - group_name, group_results = next(iter(groups_map.items())) - n_shot = n_shot_data.get(group_name, "unknown") - if n_shot == "unknown": - for subtask_name in group_subtasks_map.get(group_name, []): - if subtask_name in n_shot_data: - n_shot = n_shot_data[subtask_name] - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - performance, metric_name = _resolve_metric(group_name, group_results) - if performance is not None: - if check: - completed_jobs.add((model_name, group_name, n_shot)) - rows.append( - { - "model_name": model_name, - "task": group_name, - "n_shot": n_shot, - "performance": performance, - "metric_name": metric_name if metric_name is not None else "", - } - ) - # Skip per-task iteration when groups are present - continue - - for task_name, task_results in results.items(): - # Skip entries already added from groups - if groups_map and task_name in group_aggregate_names: - continue - # Skip any lm-eval group subtasks; keep only aggregates - if task_name in group_subtask_names: - continue - - # Skip MMLU subtasks - only keep the aggregate score - if task_name.startswith("mmlu_") and task_name != "mmlu": - continue - - # Skip Global MMLU subtasks - keep only aggregates like global_mmlu_full_pt - if task_name.startswith("global_mmlu_") and task_name.count("_") >= 4: - continue - - n_shot = n_shot_data.get(task_name, "unknown") - - # If this is a group aggregate and n_shot is missing, derive from any subtask - if task_name in group_aggregate_names and n_shot == "unknown": - for subtask_name in group_subtasks_map.get(task_name, []): - if subtask_name in n_shot_data: - n_shot = n_shot_data[subtask_name] - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - - # Special handling for MMLU aggregate - get n_shot from any MMLU subtask - if task_name == "mmlu" and n_shot == "unknown": - for key, value in n_shot_data.items(): - if key.startswith("mmlu_"): - n_shot = value - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - - # Special handling for Global MMLU aggregates - get n_shot from subtasks - if task_name.startswith("global_mmlu_") and n_shot == "unknown": - prefix = f"{task_name}_" - for key, value in n_shot_data.items(): - if key.startswith(prefix): - n_shot = value - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - - # Skip lmms-eval parent task placeholders (no numeric metrics, just alias) - if set(task_results.keys()) <= {"alias", " ", ""}: - continue - - performance, metric_name = _resolve_metric(task_name, task_results) - - if performance is not None: - if check: - completed_jobs.add((model_name, task_name, n_shot)) - - rows.append( - { - "model_name": model_name, - "task": task_name, - "n_shot": n_shot, - "performance": performance, - "metric_name": metric_name if metric_name is not None else "", - } - ) - else: - # Log missing metrics — for lmms-eval tasks this often means - # llm_as_judge_eval is null (no judge LLM configured) or the - # metric key is not yet listed in task_metrics in task-groups.yaml - logging.warning( - f"No numeric metric for '{task_name}' in {json_file.name} " - f"— value may be null (LLM judge not configured?) or metric key missing from task_metrics" - ) - - if not rows and not check: - logging.warning("No results extracted from JSON files") - return - - if rows: - df = pd.DataFrame(rows) - df.to_csv(output_csv, index=False) - logging.info(f"Results saved to {output_csv}") - logging.info(f"Extracted {len(df)} evaluation results") - - if verbose: - logging.info("Summary:") - logging.info(f"Unique models: {df['model_name'].nunique()}") - logging.info(f"Unique tasks: {df['task'].nunique()}") - logging.info( - f"N-shot values: {sorted(str(x) for x in df['n_shot'].unique())}" - ) - - if check: - logging.info("=== Evaluation Status Check ===") - - missing_jobs = [] - - for _, job in jobs_df.iterrows(): - job_tuple = (job["model_path"], job["task_path"], job["n_shot"]) - - is_completed = False - - if job_tuple in completed_jobs: - is_completed = True - else: - for completed_job in completed_jobs: - completed_model, completed_task, completed_n_shot = completed_job - - if ( - job["n_shot"] == completed_n_shot - and job["task_path"] == completed_task - and ( - str(job["model_path"]).endswith(completed_model) - or completed_model in str(job["model_path"]) - ) - ): - is_completed = True - break - - if not is_completed: - missing_jobs.append(job) - - completed_count = len(jobs_df) - len(missing_jobs) - - logging.info(f"Total scheduled jobs: {len(jobs_df)}") - logging.info(f"Completed jobs: {completed_count}") - logging.info(f"Missing jobs: {len(missing_jobs)}") - - if len(missing_jobs) > 0: - missing_df = pd.DataFrame(missing_jobs) - missing_csv = output_csv.replace(".csv", "_missing.csv") - missing_df.to_csv(missing_csv, index=False) - logging.info(f"Missing jobs saved to: {missing_csv}") - logging.info( - f"You can run these with: oellm schedule-eval --eval_csv_path {missing_csv}" - ) - - if verbose and len(missing_jobs) > 0: - logging.info("Example missing jobs:") - for _i, (_, job) in enumerate(missing_df.head(5).iterrows()): - logging.info( - f" - {job['model_path']} | {job['task_path']} | n_shot={job['n_shot']}" - ) - if len(missing_jobs) > 5: - logging.info(f" ... and {len(missing_jobs) - 5} more") - - def main(): _filter_warnings() auto_cli( diff --git a/oellm/results.py b/oellm/results.py new file mode 100644 index 00000000..429a689c --- /dev/null +++ b/oellm/results.py @@ -0,0 +1,381 @@ +"""Result collection and metric resolution for evaluation outputs.""" + +import json +import logging +from importlib.resources import files +from pathlib import Path + +import pandas as pd +import yaml + +from oellm.constants import METRIC_FALLBACK_KEYS +from oellm.utils import _setup_logging + + +def _resolve_metric( + task_name: str, result_dict: dict, task_metrics: dict +) -> tuple[float | None, str | None]: + """Return (value, metric_name) for task_name from result_dict.""" + + # Normalise lmms-eval task-scoped metric keys so lm-eval and lmms-eval + # output is handled identically. lmms-eval writes keys like + # "vqav2/vqa_score,none"; strip the "task_name/" prefix so the lookup + # below sees "vqa_score,none" regardless of engine. Keys without "/" + # (lm-eval format) are passed through unchanged. + result_dict = { + (k.split("/", 1)[1] if "/" in k else k): v for k, v in result_dict.items() + } + + def _first_numeric(d: dict, *candidates: str) -> tuple[float | None, str | None]: + for c in candidates: + if c in d and isinstance(d[c], (int, float)): + return float(d[c]), c + return None, None + + def _first_matching_prefix(d: dict, prefix: str) -> tuple[float | None, str | None]: + for k, v in d.items(): + if (k == prefix or k.startswith(prefix + ",")) and isinstance( + v, (int, float) + ): + return float(v), k + return None, None + + preferred = task_metrics.get(task_name) + if preferred is not None: + val, key = _first_numeric(result_dict, f"{preferred},none", preferred) + if val is not None: + return val, key + val, key = _first_matching_prefix(result_dict, preferred) + return val, key + + for metric in METRIC_FALLBACK_KEYS: + val, key = _first_numeric(result_dict, metric) + if val is not None: + return val, key + val, key = _first_matching_prefix(result_dict, metric.split(",")[0]) + if val is not None: + return val, key + + # Last resort: pick the first numeric non-stderr value (catches lmms-eval + # benchmarks with non-standard metric names like mme_cognition_score) + for k, v in result_dict.items(): + if ( + isinstance(v, (int, float)) + and "stderr" not in k + and k not in ("alias", " ", "") + ): + return float(v), k + return None, None + + +def _infer_global_n_shot(n_shot_data: dict) -> int | None: + """Infer a global n_shot if exactly one unique value exists.""" + try: + candidate_values = [] + for _v in n_shot_data.values(): + if isinstance(_v, (int | float)): + candidate_values.append(int(_v)) + elif isinstance(_v, str) and _v.isdigit(): + candidate_values.append(int(_v)) + unique_values = set(candidate_values) + if len(unique_values) == 1: + return next(iter(unique_values)) + except Exception: + pass + return None + + +def _resolve_n_shot( + task_name: str, + n_shot_data: dict, + group_subtasks_map: dict, + group_aggregate_names: set, + global_n_shot: int | None, +) -> int | str: + """Resolve n_shot for a task, with fallbacks for groups and MMLU.""" + n_shot = n_shot_data.get(task_name, "unknown") + + # If this is a group aggregate and n_shot is missing, derive from any subtask + if task_name in group_aggregate_names and n_shot == "unknown": + for subtask_name in group_subtasks_map.get(task_name, []): + if subtask_name in n_shot_data: + n_shot = n_shot_data[subtask_name] + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + + # Special handling for MMLU aggregate - get n_shot from any MMLU subtask + if task_name == "mmlu" and n_shot == "unknown": + for key, value in n_shot_data.items(): + if key.startswith("mmlu_"): + n_shot = value + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + + # Special handling for Global MMLU aggregates - get n_shot from subtasks + if task_name.startswith("global_mmlu_") and n_shot == "unknown": + prefix = f"{task_name}_" + for key, value in n_shot_data.items(): + if key.startswith(prefix): + n_shot = value + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + + return n_shot + + +def _load_task_metrics() -> dict: + """Load task_metrics from core YAML and all contrib suites.""" + task_groups_yaml = files("oellm.resources") / "task-groups.yaml" + with open(str(task_groups_yaml)) as _f: + _tg_cfg = yaml.safe_load(_f) + task_metrics = _tg_cfg.get("task_metrics", {}) + + from oellm.registry import ( + get_all_task_groups as _contrib_task_groups, + ) + + task_metrics.update(_contrib_task_groups().get("task_metrics", {})) + return task_metrics + + +def collect_results( + results_dir: str, + output_csv: str = "eval_results.csv", + *, + check: bool = False, + verbose: bool = False, +) -> None: + """ + Collect evaluation results from JSON files and export to CSV. + + Args: + results_dir: Path to the directory containing result JSON files + output_csv: Output CSV filename (default: eval_results.csv) + check: Check for missing evaluations and create a missing jobs CSV + verbose: Enable verbose logging + """ + _setup_logging(verbose) + + task_metrics = _load_task_metrics() + + results_path = Path(results_dir) + if not results_path.exists(): + raise ValueError(f"Results directory does not exist: {results_dir}") + + # lm-eval writes flat JSON files: results/.json + # lmms-eval writes nested dirs: results/.json//_results.json + # rglob("*.json") + is_file() finds both without breaking backward compat. + search_root = ( + (results_path / "results") + if (results_path / "results").is_dir() + else results_path + ) + json_files = [p for p in search_root.rglob("*.json") if p.is_file()] + + if not json_files: + logging.warning(f"No JSON files found in {results_dir}") + if not check: + return + + logging.info(f"Found {len(json_files)} result files") + + # If check mode, also load the jobs.csv to compare + if check: + jobs_csv_path = results_path / "jobs.csv" + if not jobs_csv_path.exists(): + logging.warning(f"No jobs.csv found in {results_dir}, cannot perform check") + check = False + else: + jobs_df = pd.read_csv(jobs_csv_path) + logging.info(f"Found {len(jobs_df)} scheduled jobs in jobs.csv") + + rows = [] + completed_jobs = set() + + for json_file in json_files: + with open(json_file) as f: + data = json.load(f) + + # lmms-eval sets model_name to the adapter type (e.g. "llava_hf"), + # not the checkpoint path; the actual path is in model_name_or_path. + model_name = data.get("model_name_or_path") or data.get("model_name", "unknown") + + results = data.get("results", {}) + n_shot_data = data.get("n-shot", {}) + + # lmms-eval has no "n-shot" dict; fall back to per-task config "num_fewshot" + if not n_shot_data: + for _task, _cfg in data.get("configs", {}).items(): + if isinstance(_cfg, dict): + shot = _cfg.get("num_fewshot") + if shot is not None: + n_shot_data[_task] = shot + + global_n_shot = _infer_global_n_shot(n_shot_data) + + # Aggregate groups (lm-eval harness) + groups_map = data.get("groups", {}) + group_subtasks_map = data.get("group_subtasks", {}) + group_aggregate_names = set(groups_map.keys()) | set(group_subtasks_map.keys()) + group_subtask_names: set[str] = set() + for _agg, _subs in group_subtasks_map.items(): + for _s in _subs: + group_subtask_names.add(_s) + + # Prefer only the first aggregate metric from groups (simplified) + if groups_map: + group_name, group_results = next(iter(groups_map.items())) + n_shot = n_shot_data.get(group_name, "unknown") + if n_shot == "unknown": + for subtask_name in group_subtasks_map.get(group_name, []): + if subtask_name in n_shot_data: + n_shot = n_shot_data[subtask_name] + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + performance, metric_name = _resolve_metric( + group_name, group_results, task_metrics + ) + if performance is not None: + if check: + completed_jobs.add((model_name, group_name, n_shot)) + rows.append( + { + "model_name": model_name, + "task": group_name, + "n_shot": n_shot, + "performance": performance, + "metric_name": metric_name if metric_name is not None else "", + } + ) + # Skip per-task iteration when groups are present + continue + + for task_name, task_results in results.items(): + # Skip entries already added from groups + if groups_map and task_name in group_aggregate_names: + continue + # Skip any lm-eval group subtasks; keep only aggregates + if task_name in group_subtask_names: + continue + + # Skip MMLU subtasks - only keep the aggregate score + if task_name.startswith("mmlu_") and task_name != "mmlu": + continue + + # Skip Global MMLU subtasks - keep only aggregates like global_mmlu_full_pt + if task_name.startswith("global_mmlu_") and task_name.count("_") >= 4: + continue + + n_shot = _resolve_n_shot( + task_name, + n_shot_data, + group_subtasks_map, + group_aggregate_names, + global_n_shot, + ) + + # Skip lmms-eval parent task placeholders (no numeric metrics, just alias) + if set(task_results.keys()) <= {"alias", " ", ""}: + continue + + performance, metric_name = _resolve_metric( + task_name, task_results, task_metrics + ) + + if performance is not None: + if check: + completed_jobs.add((model_name, task_name, n_shot)) + + rows.append( + { + "model_name": model_name, + "task": task_name, + "n_shot": n_shot, + "performance": performance, + "metric_name": metric_name if metric_name is not None else "", + } + ) + else: + # Log missing metrics — for lmms-eval tasks this often means + # llm_as_judge_eval is null (no judge LLM configured) or the + # metric key is not yet listed in task_metrics in task-groups.yaml + logging.warning( + f"No numeric metric for '{task_name}' in {json_file.name} " + f"— value may be null (LLM judge not configured?) or metric key missing from task_metrics" + ) + + if not rows and not check: + logging.warning("No results extracted from JSON files") + return + + if rows: + df = pd.DataFrame(rows) + df.to_csv(output_csv, index=False) + logging.info(f"Results saved to {output_csv}") + logging.info(f"Extracted {len(df)} evaluation results") + + if verbose: + logging.info("Summary:") + logging.info(f"Unique models: {df['model_name'].nunique()}") + logging.info(f"Unique tasks: {df['task'].nunique()}") + logging.info( + f"N-shot values: {sorted(str(x) for x in df['n_shot'].unique())}" + ) + + if check: + logging.info("=== Evaluation Status Check ===") + + missing_jobs = [] + + for _, job in jobs_df.iterrows(): + job_tuple = (job["model_path"], job["task_path"], job["n_shot"]) + + is_completed = False + + if job_tuple in completed_jobs: + is_completed = True + else: + for completed_job in completed_jobs: + completed_model, completed_task, completed_n_shot = completed_job + + if ( + job["n_shot"] == completed_n_shot + and job["task_path"] == completed_task + and ( + str(job["model_path"]).endswith(completed_model) + or completed_model in str(job["model_path"]) + ) + ): + is_completed = True + break + + if not is_completed: + missing_jobs.append(job) + + completed_count = len(jobs_df) - len(missing_jobs) + + logging.info(f"Total scheduled jobs: {len(jobs_df)}") + logging.info(f"Completed jobs: {completed_count}") + logging.info(f"Missing jobs: {len(missing_jobs)}") + + if len(missing_jobs) > 0: + missing_df = pd.DataFrame(missing_jobs) + missing_csv = output_csv.replace(".csv", "_missing.csv") + missing_df.to_csv(missing_csv, index=False) + logging.info(f"Missing jobs saved to: {missing_csv}") + logging.info( + f"You can run these with: oellm schedule-eval --eval_csv_path {missing_csv}" + ) + + if verbose and len(missing_jobs) > 0: + logging.info("Example missing jobs:") + for _i, (_, job) in enumerate(missing_df.head(5).iterrows()): + logging.info( + f" - {job['model_path']} | {job['task_path']} | n_shot={job['n_shot']}" + ) + if len(missing_jobs) > 5: + logging.info(f" ... and {len(missing_jobs) - 5} more") diff --git a/oellm/utils.py b/oellm/utils.py index 8b254c0e..5bd74dbf 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -165,6 +165,23 @@ def __missing__(self, key): for k, v in final_env.items(): os.environ.setdefault(k, v) + # Validate that critical sbatch variables resolved to real values. + _required_vars = [ + "PARTITION", + "ACCOUNT", + "EVAL_BASE_DIR", + "EVAL_OUTPUT_DIR", + "GPUS_PER_NODE", + ] + missing = [ + v for v in _required_vars if not os.environ.get(v) or "{" in os.environ.get(v, "") + ] + if missing: + raise RuntimeError( + f"Required cluster variables are missing or unresolved: {', '.join(missing)}. " + f"Check your clusters.yaml entry or set them in your environment." + ) + def _num_jobs_in_queue() -> int: user = os.environ.get("USER") @@ -280,7 +297,9 @@ def _process_model_paths(models: Iterable[str]): try: snapshot_download( repo_id=repo_id, - cache_dir=Path(os.getenv("HF_HOME")) / "hub", + cache_dir=Path(os.getenv("HF_HOME")) / "hub" + if "HF_HOME" in os.environ + else None, **snapshot_kwargs, ) per_model_paths.append(model) From d245bc5d2e1eb67f02fd4c6269008abc06b2e058 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Wed, 25 Mar 2026 11:40:50 +0100 Subject: [PATCH 10/44] [Base][Refactoring] Refresh readme 1. Add new documentation table grouped by category (Cluster Setup, Environment & Infrastructure, Extending the Platform) 2. Add oellm/contrib/README.md as a registry of community-contributed benchmarks, starting with RegionReasoner 3. Link contrib registry and contributing guide from main README --- README.md | 185 +++++++++++----------------------------- oellm/contrib/README.md | 24 ++++++ 2 files changed, 74 insertions(+), 135 deletions(-) create mode 100644 oellm/contrib/README.md diff --git a/README.md b/README.md index 215e28c5..7c8a28d5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ELLIOT Evaluation Platform -A multimodal evaluation framework for scheduling LLM and VLM evaluations across HPC clusters. Extends the original oellm-cli with image modality support and a plugin interface for adding new benchmarks and modalities. +A multimodal evaluation framework for scheduling LLM and VLM evaluations across HPC clusters. Built as an orchestration layer over [lm-eval](https://github.com/EleutherAI/lm-evaluation-harness), [lighteval](https://github.com/huggingface/lighteval), and [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval), with a plugin system for contributing custom benchmarks. ## Features @@ -8,39 +8,35 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Collect results** and check for missing evaluations: `oellm collect-results` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA) -- **Image evaluation** via [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) -- **Plugin interface** (`BaseTask` / `BaseMetric` / `BaseModelAdapter`) for adding new benchmarks without touching core scheduling logic -- **Automatic building and deployment of containers** +- **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) +- **Plugin system** for contributing custom benchmarks without touching core code +- **Automatic container builds** via GitHub Actions ## Quick Start **Prerequisites:** - Install [uv](https://docs.astral.sh/uv/#installation) -- Set the `HF_HOME` environment variable to point to your HuggingFace cache directory (e.g. `export HF_HOME="/path/to/your/hf_home"`). This is where models and datasets will be cached. Compute nodes typically have no internet access, so all assets must be pre-downloaded into this directory. +- Set `HF_HOME` to your HuggingFace cache directory (e.g. `export HF_HOME="/path/to/hf_home"`) ```bash -# Install the package +# Install uv tool install -p 3.12 git+https://github.com/elliot-project/elliot-cli.git -# Run evaluations using a task group (recommended) +# Run evaluations using a task group oellm schedule-eval \ - --models "microsoft/DialoGPT-medium,EleutherAI/pythia-160m" \ + --models "EleutherAI/pythia-160m" \ --task_groups "open-sci-0.01" -# Or specify individual tasks +# Image evaluation (requires venv with lmms-eval) oellm schedule-eval \ - --models "EleutherAI/pythia-160m" \ - --tasks "hellaswag,mmlu" \ - --n_shot 5 + --models "llava-hf/llava-1.5-7b-hf" \ + --task_groups "image-vqa" \ + --venv_path ~/elliot-venv ``` -This will automatically: -- Detect your current HPC cluster (Leonardo, LUMI, or JURECA) -- Download and cache the specified models -- Pre-download datasets for known tasks (see warning below) -- Generate and submit a SLURM job array with appropriate cluster-specific resources and using containers built for this cluster +This will automatically detect your cluster, download models and datasets, and submit a SLURM job array with cluster-specific resources. -In case you do not want to rely on the containers provided on a given cluster or try out specific package versions, you can use a custom environment by passing `--venv_path`, see [docs/VENV.md](docs/VENV.md). +For custom environments instead of containers, pass `--venv_path` (see [docs/VENV.md](docs/VENV.md)). ## Task Groups @@ -52,15 +48,14 @@ Task groups are pre-defined evaluation suites in [`task-groups.yaml`](oellm/reso |---|---|---| | `open-sci-0.01` | COPA, MMLU, HellaSwag, ARC, etc. | lm-eval | | `belebele-eu-5-shot` | Belebele in 23 European languages | lm-eval | -| `flores-200-eu-to-eng` | EU → English translation | lighteval | -| `flores-200-eng-to-eu` | English → EU translation | lighteval | +| `flores-200-eu-to-eng` | EU to English translation | lighteval | +| `flores-200-eng-to-eu` | English to EU translation | lighteval | | `global-mmlu-eu` | Global MMLU in EU languages | lm-eval | | `mgsm-eu` | Multilingual GSM8K | lm-eval | | `generic-multilingual` | XWinograd, XCOPA, XStoryCloze | lm-eval | | `include` | INCLUDE benchmarks (44 languages) | lm-eval | -Super groups: -- `oellm-multilingual` — all multilingual benchmarks combined +Super groups: `oellm-multilingual` (all multilingual benchmarks combined) ### Image @@ -76,94 +71,45 @@ Super groups: | `image-ocrbench` | OCRBench | lmms-eval | | `image-mathvista` | MathVista | lmms-eval | -Image evaluation requires a venv with `lmms-eval` installed (see [docs/VENV.md](docs/VENV.md)). The lmms-eval adapter class (`llava_hf`, `qwen2_5_vl`, etc.) is auto-detected from the model name — no extra configuration needed. +The lmms-eval adapter class (`llava_hf`, `qwen2_5_vl`, etc.) is auto-detected from the model name. + +### Custom Benchmarks (contrib) + +Community-contributed benchmarks that run outside the standard evaluation engines. See the [contrib registry](oellm/contrib/README.md) for the full list. ```bash -# Run all 8 image benchmarks at once +# Run all 8 image benchmarks oellm schedule-eval \ --models "llava-hf/llava-1.5-7b-hf" \ --task_groups "image-vqa" \ --venv_path ~/elliot-venv -# Smoke-test a single benchmark (fast, use --limit for a few samples) -oellm schedule-eval \ - --models "llava-hf/llava-1.5-7b-hf" \ - --task_groups "image-mathvista" \ - --venv_path ~/elliot-venv \ - --limit 10 - # Mix image and text benchmarks in one submission oellm schedule-eval \ --models "llava-hf/llava-1.5-7b-hf" \ --task_groups "image-mmbench,open-sci-0.01" \ --venv_path ~/elliot-venv -``` - -```bash -# Use a task group -oellm schedule-eval --models "model-name" --task_groups "open-sci-0.01" -# Use multiple task groups +# Use multiple task groups or a super group oellm schedule-eval --models "model-name" --task_groups "belebele-eu-5-shot,global-mmlu-eu" - -# Use a super group oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" ``` -## SLURM Overrides - -Override cluster defaults (partition, account, time limit, etc.) with `--slurm_template_var` (JSON object): - -```bash -# Use a different partition (e.g. dev-g on LUMI when small-g is crowded) -oellm schedule-eval --models "model-name" --task_groups "open-sci-0.01" \ - --slurm_template_var '{"PARTITION":"dev-g"}' - -# Multiple overrides: partition, account, time limit, GPUs -oellm schedule-eval --models "model-name" --task_groups "open-sci-0.01" \ - --slurm_template_var '{"PARTITION":"dev-g","ACCOUNT":"myproject","TIME":"02:00:00","GPUS_PER_NODE":2}' -``` - -Use exact env var names: `PARTITION`, `ACCOUNT`, `GPUS_PER_NODE`. `TIME` (HH:MM:SS) overrides the time limit. - -## ⚠️ Dataset Pre-Download Warning - -**Datasets are only automatically pre-downloaded for tasks defined in [`task-groups.yaml`](oellm/resources/task-groups.yaml).** - -If you use custom tasks via `--tasks` that are not in the task groups registry, the CLI will attempt to look them up but **cannot guarantee the datasets will be cached**. This may cause failures on compute nodes that don't have network access. - -**Recommendation:** Use `--task_groups` when possible, or ensure your custom task datasets are already cached in `$HF_HOME` before scheduling. - ## Collecting Results -After evaluations complete, collect results into a CSV: - ```bash # Basic collection oellm collect-results /path/to/eval-output-dir # Check for missing evaluations and create a CSV for re-running them oellm collect-results /path/to/eval-output-dir --check --output_csv results.csv -``` -The `--check` flag compares completed results against `jobs.csv` and outputs a `results_missing.csv` that can be used to re-schedule failed jobs: - -```bash +# Re-schedule failed jobs oellm schedule-eval --eval_csv_path results_missing.csv ``` -## CSV-Based Scheduling - -For full control, provide a CSV file with columns: `model_path`, `task_path`, `n_shot`, and optionally `eval_suite`: - -```bash -oellm schedule-eval --eval_csv_path custom_evals.csv -``` - ## Installation -### General Installation - ```bash uv tool install -p 3.12 git+https://github.com/elliot-project/elliot-cli.git ``` @@ -173,34 +119,11 @@ Update to latest: uv tool upgrade oellm ``` -### JURECA/JSC Specifics - -Due to limited space in `$HOME` on JSC clusters, set these environment variables: - -```bash -export UV_CACHE_DIR="/p/project1//$USER/.cache/uv-cache" -export UV_INSTALL_DIR="/p/project1//$USER/.local" -export UV_PYTHON_INSTALL_DIR="/p/project1//$USER/.local/share/uv/python" -export UV_TOOL_DIR="/p/project1//$USER/.cache/uv-tool-cache" -``` - -## Supported Clusters - -We support: Leonardo, LUMI, and JURECA - -Cluster-specific access guides: -- [Leonardo HPC](docs/LEONARDO.md) - -## CLI Options - -```bash -oellm schedule-eval --help -``` +For cluster-specific setup, see the [documentation](#documentation) section. ## Development ```bash -# Clone and install in dev mode git clone https://github.com/elliot-project/elliot-cli.git cd elliot-cli uv sync --extra dev @@ -208,51 +131,43 @@ uv sync --extra dev # Run all unit tests uv run pytest tests/ -v -# Run dataset validation tests (requires network access) -uv run pytest tests/test_datasets.py -v - # Download-only mode for testing uv run oellm schedule-eval --models "EleutherAI/pythia-160m" --task_groups "open-sci-0.01" --download_only ``` -## Plugin Interface +## Documentation -The `oellm.core` package provides abstract base classes for extending the platform without modifying core scheduling logic: +### Cluster Setup -```python -from oellm.core import BaseTask, BaseMetric, BaseModelAdapter -from oellm.task_groups import DatasetSpec +| Cluster | Guide | +|---|---| +| Leonardo (CINECA) | [docs/LEONARDO.md](docs/LEONARDO.md) | +| LUMI, JURECA | Coming soon | -# Register a new benchmark (one-liner if it's already in lmms-eval) -class MyTask(BaseTask): - @property - def name(self) -> str: - return "my_benchmark" +### Environment & Infrastructure - @property - def suite(self) -> str: - return "lmms_eval" # or "lm_eval" / "lighteval" - - @property - def n_shots(self) -> list[int]: - return [0] - - @property - def dataset_specs(self) -> list[DatasetSpec]: - return [DatasetSpec(repo_id="org/my-dataset")] -``` +| Doc | Description | +|---|---| +| [Using a Virtual Environment](docs/VENV.md) | Setting up a custom venv with lm-eval, lmms-eval, and lighteval | +| [Container Workflow](docs/CONTAINERS.md) | How Apptainer containers are built, deployed, and used | -See `oellm/core/` for full interface documentation. +### Extending the Platform -## Deploying containers +| Doc | Description | +|---|---| +| [Adding Tasks & Task Groups](docs/TASKS.md) | YAML structure for defining new evaluation suites | +| [Contributing Custom Benchmarks](oellm/contrib/CONTRIBUTING.md) | Step-by-step guide for adding a contrib plugin | +| [Contrib Registry](oellm/contrib/README.md) | List of community-contributed benchmarks | -Containers are deployed manually since [PR #46](https://github.com/elliot-project/elliot-cli/pull/46) to save costs. +## Contributing Custom Benchmarks -To build and deploy them, select run workflow in [Actions](https://github.com/elliot-project/elliot-cli/actions/workflows/build-and-push-apptainer.yml). +ELLIOT supports two paths for adding benchmarks: +1. **Benchmark already in lm-eval / lighteval / lmms-eval** -- add a YAML entry to [`task-groups.yaml`](oellm/resources/task-groups.yaml) +2. **Fully custom benchmark** -- drop a contrib plugin into [`oellm/contrib/`](oellm/contrib/) -## Troubleshooting +See the [Contributing Guide](oellm/contrib/CONTRIBUTING.md) for step-by-step instructions. -**HuggingFace quota issues**: Ensure you're logged in with `HF_TOKEN` and are part of the [OpenEuroLLM](https://huggingface.co/OpenEuroLLM) organization. +## Deploying Containers -**Dataset download failures on compute nodes**: Use `--task_groups` for automatic dataset caching, or pre-download datasets manually before scheduling. +Containers are deployed manually since [PR #46](https://github.com/elliot-project/elliot-cli/pull/46). To build and deploy, select "Run workflow" in [Actions](https://github.com/elliot-project/elliot-cli/actions/workflows/build-and-push-apptainer.yml). diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md new file mode 100644 index 00000000..8db5f05f --- /dev/null +++ b/oellm/contrib/README.md @@ -0,0 +1,24 @@ +# Contrib Benchmark Registry + +Community-contributed benchmarks integrated into the ELLIOT evaluation platform. Each benchmark runs as a self-contained plugin -- no changes to core scheduling code required. + +To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). + +## Benchmarks + +| Benchmark | Task Group | Description | Paper | Code | +|---|---|---|---|---| +| RegionReasoner | `region-reasoner` | Multi-turn region grounding and segmentation on RefCOCOg. Evaluates a model's ability to locate and segment objects described in multi-turn conversations. | [arXiv:2602.03733](https://arxiv.org/abs/2602.03733) | [lmsdss/RegionReasoner](https://github.com/lmsdss/RegionReasoner) | + +### RegionReasoner + +**Metrics:** gIoU (primary), cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9 + +```bash +oellm schedule-eval \ + --models lmsdss/RegionReasoner-7B \ + --task_groups region-reasoner \ + --venv_path ~/elliot-venv +``` + +Requires cluster-specific setup (`REGION_REASONER_DIR`, etc.). See the full [RegionReasoner README](region_reasoner/README.md) for prerequisites and configuration. From f7ac14a67d078161775de9d59cd8fd877c8360e3 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Sun, 29 Mar 2026 10:40:47 +0200 Subject: [PATCH 11/44] [Contrib][Custom Benchmark] Fix RegionDial-Bench OOM and multi-GPU sharding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add --mem=0 to SLURM template for full node memory allocation - Implement streaming pre-sharding - Fix CUDA_VISIBLE_DEVICES assignment per shard - Fix dataset download to fetch all splits (refcocog + refcocoplus) - Rename region_reasoner → regiondial_bench to match paper terminology - Allow running splits independently (regiondial-refcocog, regiondial-refcocoplus) --- docs/TASKS.md | 48 +-- oellm/contrib/CONTRIBUTING.md | 4 +- oellm/contrib/README.md | 10 +- oellm/contrib/dispatch.py | 6 +- oellm/contrib/region_reasoner/__init__.py | 2 - oellm/contrib/region_reasoner/suite.py | 275 ------------ oellm/contrib/region_reasoner/task.py | 50 --- .../README.md | 48 ++- oellm/contrib/regiondial_bench/__init__.py | 1 + .../adapter.py | 4 +- .../metrics.py | 9 +- oellm/contrib/regiondial_bench/suite.py | 399 ++++++++++++++++++ oellm/contrib/regiondial_bench/task.py | 107 +++++ oellm/core/base_task.py | 2 +- oellm/main.py | 10 +- oellm/registry.py | 2 +- oellm/resources/template.sbatch | 4 +- oellm/task_groups.py | 19 +- pyproject.toml | 1 + ...n_reasoner.py => test_regiondial_bench.py} | 362 +++++++++++----- tests/test_registry.py | 34 +- 21 files changed, 872 insertions(+), 525 deletions(-) delete mode 100644 oellm/contrib/region_reasoner/__init__.py delete mode 100644 oellm/contrib/region_reasoner/suite.py delete mode 100644 oellm/contrib/region_reasoner/task.py rename oellm/contrib/{region_reasoner => regiondial_bench}/README.md (65%) create mode 100644 oellm/contrib/regiondial_bench/__init__.py rename oellm/contrib/{region_reasoner => regiondial_bench}/adapter.py (90%) rename oellm/contrib/{region_reasoner => regiondial_bench}/metrics.py (94%) create mode 100644 oellm/contrib/regiondial_bench/suite.py create mode 100644 oellm/contrib/regiondial_bench/task.py rename tests/{test_region_reasoner.py => test_regiondial_bench.py} (59%) diff --git a/docs/TASKS.md b/docs/TASKS.md index 329db86c..86275028 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -75,12 +75,8 @@ Run with: oellm schedule-eval --models "path/to/vlm" --task_groups "my-image-benchmark" ``` -The model adapter defaults to `llava_hf`. Override via `--slurm_template_var`: - -```bash -oellm schedule-eval --models "path/to/vlm" --task_groups "my-image-benchmark" \ - --slurm_template_var '{"LMMS_MODEL_TYPE":"qwen_vl_chat"}' -``` +The lmms-eval model adapter (e.g. `llava_hf`, `qwen2_vl`) is auto-detected +from the model name. No manual override is needed. ## Field Reference @@ -101,38 +97,10 @@ oellm schedule-eval --models "path/to/vlm" --task_groups "my-image-benchmark" \ Tasks without a `dataset` field will not have their data pre-downloaded and are not covered by CI validation. -## Plugin Interface (Advanced) - -For programmatic task registration without editing the YAML, use the `BaseTask` abstract base class from `oellm.core`: - -```python -from oellm.core import BaseTask -from oellm.task_groups import DatasetSpec - -class MyImageTask(BaseTask): - @property - def name(self) -> str: - return "my_benchmark" # canonical name used in CSV scheduling - - @property - def suite(self) -> str: - return "lmms_eval" # or "lm_eval" / "lighteval" - - @property - def n_shots(self) -> list[int]: - return [0] - - @property - def dataset_specs(self) -> list[DatasetSpec]: - return [DatasetSpec(repo_id="org/my-dataset")] -``` - -Override `engine_task_name` if the engine uses a different name than `name`: - -```python - @property - def engine_task_name(self) -> str: - return "my_benchmark_v2" # passed to --tasks; defaults to self.name -``` +## Custom Benchmarks (contrib plugins) -See `oellm/core/` for the full `BaseTask`, `BaseMetric`, and `BaseModelAdapter` interfaces. +For benchmarks that have their own inference scripts, custom metrics, or +are not part of lm-eval / lighteval / lmms-eval, use the contrib plugin +system. See [`oellm/contrib/CONTRIBUTING.md`](../oellm/contrib/CONTRIBUTING.md) +for the full guide and `oellm/contrib/regiondial_bench/` as a reference +implementation. diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index 61947b57..46927812 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -56,7 +56,7 @@ oellm/contrib/my_suite/ └── README.md ``` -See `oellm/contrib/region_reasoner/` as a complete reference. +See `oellm/contrib/regiondial_bench/` as a complete reference. --- @@ -261,7 +261,7 @@ Add `tests/test_my_suite.py`. Cover at minimum: - Task group expansion: correct `(task, n_shot, suite)` tuples - Dry-run `schedule_evals()` produces SBATCH with `oellm.contrib.dispatch` -See `tests/test_region_reasoner.py` as a reference. +See `tests/test_regiondial_bench.py` as a reference. --- diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md index 8db5f05f..21250a6b 100644 --- a/oellm/contrib/README.md +++ b/oellm/contrib/README.md @@ -8,17 +8,17 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). | Benchmark | Task Group | Description | Paper | Code | |---|---|---|---|---| -| RegionReasoner | `region-reasoner` | Multi-turn region grounding and segmentation on RefCOCOg. Evaluates a model's ability to locate and segment objects described in multi-turn conversations. | [arXiv:2602.03733](https://arxiv.org/abs/2602.03733) | [lmsdss/RegionReasoner](https://github.com/lmsdss/RegionReasoner) | +| RegionDial-Bench | `regiondial-bench` | Multi-round region grounding and segmentation on RefCOCOg and RefCOCO+. Evaluates robustness to error accumulation across dialogue turns. | [arXiv:2602.03733](https://arxiv.org/abs/2602.03733) | [lmsdss/RegionReasoner](https://github.com/lmsdss/RegionReasoner) | -### RegionReasoner +### RegionDial-Bench -**Metrics:** gIoU (primary), cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9 +**Metrics:** gIoU (primary), cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9, per-round R1–R7 ```bash oellm schedule-eval \ --models lmsdss/RegionReasoner-7B \ - --task_groups region-reasoner \ + --task_groups regiondial-bench \ --venv_path ~/elliot-venv ``` -Requires cluster-specific setup (`REGION_REASONER_DIR`, etc.). See the full [RegionReasoner README](region_reasoner/README.md) for prerequisites and configuration. +Requires cluster-specific setup (`REGION_REASONER_DIR`, etc.). See the full [RegionDial-Bench README](regiondial_bench/README.md) for prerequisites and configuration. diff --git a/oellm/contrib/dispatch.py b/oellm/contrib/dispatch.py index de4d4b70..00285058 100644 --- a/oellm/contrib/dispatch.py +++ b/oellm/contrib/dispatch.py @@ -3,14 +3,14 @@ Called from template.sbatch's ``*)`` catch-all case as:: python -m oellm.contrib.dispatch \\ - --suite "region_reasoner:vision_reasoner" \\ + --suite "regiondial_bench:vision_reasoner" \\ --model_path "/path/to/model" \\ - --task "regionreasoner_refcocog" \\ + --task "regiondial_refcocog" \\ --n_shot 0 \\ --output_path "/evals/dir/abc123.json" The suite name may include a model-flags suffix separated by ``:``, e.g. -``region_reasoner:vision_reasoner``. The suffix is passed to ``suite.run()`` +``regiondial_bench:vision_reasoner``. The suffix is passed to ``suite.run()`` as ``model_flags``. """ diff --git a/oellm/contrib/region_reasoner/__init__.py b/oellm/contrib/region_reasoner/__init__.py deleted file mode 100644 index f581d4ea..00000000 --- a/oellm/contrib/region_reasoner/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# RegionReasoner contrib package. -# See suite.py for the plugin protocol implementation. diff --git a/oellm/contrib/region_reasoner/suite.py b/oellm/contrib/region_reasoner/suite.py deleted file mode 100644 index 90331c10..00000000 --- a/oellm/contrib/region_reasoner/suite.py +++ /dev/null @@ -1,275 +0,0 @@ -"""RegionReasoner contrib suite — plugin protocol implementation. - -This module follows the plugin protocol defined in ``oellm/registry.py``. -It is the reference implementation for custom benchmark integration. - -Cluster setup -------------- -The following environment variables must be set in ``clusters.yaml`` (or the -cluster's module/profile system) before using the ``region-reasoner`` task group: - -``REGION_REASONER_DIR`` - Absolute path to a local clone of the RegionReasoner repository - (https://github.com/lmsdss/RegionReasoner). The eval scripts - ``test/evaluation/evaluation_multi_segmentation.py`` must be present. - -``REGION_REASONER_TEST_JSON`` *(optional)* - Absolute path to the test JSON file (``refcocog_multi_turn.json``). - If not set, the file is downloaded automatically from - ``lmsdss/regionreasoner_test_data`` on the HF Hub before inference. - -``REGION_REASONER_NUM_GPUS`` *(optional, default: 4)* - Number of GPU shards to use for parallel inference. Should match the - number of GPUs available on the compute node. - -Output format -------------- -``run()`` writes a **lmms-eval-compatible JSON** file so that -``oellm.main.collect_results()`` works without modification:: - - { - "model_name_or_path": "", - "results": { - "regionreasoner_refcocog": { - "gIoU": 0.42, "cIoU": 0.45, "bbox_AP": 0.38, - "pass_rate_0.3": 0.71, "pass_rate_0.5": 0.55, - "pass_rate_0.7": 0.31, "pass_rate_0.9": 0.08 - } - }, - "configs": { - "regionreasoner_refcocog": {"num_fewshot": 0} - } - } -""" - -from __future__ import annotations - -import json -import logging -import subprocess -import tempfile -from pathlib import Path - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Plugin protocol: required constants -# --------------------------------------------------------------------------- - -SUITE_NAME = "region_reasoner" - -CLUSTER_ENV_VARS = [ - "REGION_REASONER_DIR", -] - -from oellm.contrib.region_reasoner.task import RegionReasonerTask # noqa: E402 - -TASK_GROUPS: dict = RegionReasonerTask.to_task_groups_dict() - -# --------------------------------------------------------------------------- -# Plugin protocol: optional — model-flag detection -# --------------------------------------------------------------------------- - - -def detect_model_flags(model_path: str) -> str | None: - """Delegate to RegionReasonerModelAdapter.to_contrib_flags().""" - from oellm.contrib.region_reasoner.adapter import RegionReasonerModelAdapter - - return RegionReasonerModelAdapter(model_path).to_contrib_flags() - - -# --------------------------------------------------------------------------- -# Plugin protocol: required — run evaluation -# --------------------------------------------------------------------------- - - -def run( - *, - model_path: str, - task: str, - n_shot: int, - output_path: Path, - model_flags: str | None, - env: dict[str, str], -) -> None: - """Execute the RegionReasoner evaluation and write results to *output_path*. - - This function: - - 1. Runs ``evaluation_multi_segmentation.py`` in parallel GPU shards. - 2. Computes gIoU, cIoU, bbox_AP, and pass_rate at four thresholds using - the :mod:`oellm.contrib.region_reasoner.metrics` implementations. - 3. Writes a lmms-eval-compatible JSON to *output_path*. - - Args: - model_path: Path or HF repo ID of the model checkpoint. - task: Task name (e.g. ``"regionreasoner_refcocog"``). - n_shot: Number of few-shot examples (always 0 for this benchmark). - output_path: Where to write the results JSON. - model_flags: Model type string (e.g. ``"vision_reasoner"``). - env: Environment variables dict (from ``os.environ``). - """ - rr_dir = env.get("REGION_REASONER_DIR", "") - num_gpus = int(env.get("REGION_REASONER_NUM_GPUS", "4")) - num_parts = int(env.get("REGION_REASONER_NUM_PARTS", str(num_gpus))) - model_type = model_flags or "vision_reasoner" - - if not rr_dir: - raise RuntimeError( - "REGION_REASONER_DIR must be set. Add it to clusters.yaml for this cluster." - ) - - test_json = env.get("REGION_REASONER_TEST_JSON") - if not test_json: - from huggingface_hub import snapshot_download - - local_dir = snapshot_download( - repo_id="lmsdss/regionreasoner_test_data", - repo_type="dataset", - allow_patterns=["raw/refcocog_multi_turn.json"], - cache_dir=Path(env["HF_HOME"]) / "hub" if "HF_HOME" in env else None, - ) - test_json = str(Path(local_dir) / "raw" / "refcocog_multi_turn.json") - - inference_script = ( - Path(rr_dir) / "test" / "evaluation" / "evaluation_multi_segmentation.py" - ) - if not inference_script.exists(): - raise FileNotFoundError( - f"RegionReasoner inference script not found: {inference_script}\n" - f"Check that REGION_REASONER_DIR={rr_dir!r} points to a valid clone." - ) - - with tempfile.TemporaryDirectory(prefix="rr_shards_") as tmp_dir: - procs = [] - for idx in range(num_gpus): - shard_env = dict(env) - shard_env["CUDA_VISIBLE_DEVICES"] = str(idx) - cmd = [ - "python", - str(inference_script), - "--model_path", - model_path, - "--model", - model_type, - "--test_data_path", - test_json, - "--output_path", - tmp_dir, # script writes /output_{idx}.json - "--vis_output_path", - str(Path(tmp_dir) / f"vis_{idx}"), - "--idx", - str(idx), - "--num_parts", - str(num_parts), - "--batch_size", - "2", - "--task_router_model_path", - "Ricky06662/TaskRouter-1.5B", - ] - logger.info("Starting shard %d/%d: %s", idx + 1, num_gpus, " ".join(cmd)) - proc = subprocess.Popen(cmd, env=shard_env, cwd=str(Path(test_json).parent)) - procs.append(proc) - - for idx, proc in enumerate(procs): - ret = proc.wait() - if ret != 0: - raise RuntimeError( - f"RegionReasoner inference shard {idx} exited with code {ret}" - ) - - logger.info("All %d shards completed. Computing metrics.", num_gpus) - - metrics = _aggregate_shards(tmp_dir) - - result_json = { - "model_name_or_path": model_path, - "results": {task: metrics}, - "configs": {task: {"num_fewshot": n_shot}}, - } - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w") as f: - json.dump(result_json, f, indent=2) - logger.info("Results written to %s", output_path) - - -def _aggregate_shards(shard_dir: str) -> dict[str, float]: - """Read per-shard output files and compute all metrics via :mod:`metrics`. - - Each shard file contains a list of per-sample dicts with pre-computed - ``intersection``, ``union``, and ``bbox_iou`` fields written by the - upstream ``evaluation_multi_segmentation.py`` script. - - All metrics are computed through the :class:`BaseMetric` subclasses in - ``oellm.contrib.region_reasoner.metrics``. - - Returns a flat dict of ``{metric_name: value}`` for all seven metrics. - """ - from oellm.contrib.region_reasoner.metrics import ( - BboxAP, - CIoU, - GIoU, - PassRate, - ) - - shard_files = sorted(Path(shard_dir).glob("output_*.json")) - if not shard_files: - raise RuntimeError( - f"No shard output files found in {shard_dir!r}. " - "The inference script may have failed silently." - ) - - samples: list[str] = [] - for shard_file in shard_files: - with open(shard_file) as f: - shard_data = json.load(f) - for sample in shard_data: - samples.append(json.dumps(sample)) - - if not samples: - raise RuntimeError( - "No samples found across shard files. " - "The inference script produced empty output." - ) - logger.info("Aggregating %d samples from %d shards", len(samples), len(shard_files)) - - empty_refs = [""] * len(samples) - all_metrics = [ - GIoU(), - CIoU(), - BboxAP(), - PassRate(0.3), - PassRate(0.5), - PassRate(0.7), - PassRate(0.9), - ] - - metrics = {} - for m in all_metrics: - val = m.compute(samples, empty_refs) - metrics[m.name] = val - logger.debug("%s = %.4f", m.name, val) - return metrics - - -# --------------------------------------------------------------------------- -# Plugin protocol: required — parse results JSON -# --------------------------------------------------------------------------- - - -def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: - """Try to parse *data* as a region_reasoner output JSON. - - Returns ``(model_id, task_name, n_shot, {metric: value})`` if the JSON - matches this suite's format, otherwise ``None``. - - Detection heuristic: the ``results`` dict contains a key that starts with - ``"regionreasoner_"`` and the value dict contains ``"gIoU"``. - """ - results = data.get("results", {}) - for task_name, task_results in results.items(): - if task_name.startswith("regionreasoner_") and "gIoU" in task_results: - model_id = data.get("model_name_or_path") or data.get("model_name", "unknown") - n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) - return model_id, task_name, int(n_shot), task_results - return None diff --git a/oellm/contrib/region_reasoner/task.py b/oellm/contrib/region_reasoner/task.py deleted file mode 100644 index 55a74544..00000000 --- a/oellm/contrib/region_reasoner/task.py +++ /dev/null @@ -1,50 +0,0 @@ -"""RegionReasoner task definition.""" - -from oellm.core.base_task import BaseTask - - -class RegionReasonerTask(BaseTask): - """Multi-turn region grounding benchmark on RefCOCOg.""" - - @property - def name(self) -> str: - return "regionreasoner_refcocog" - - @property - def suite(self) -> str: - return "region_reasoner" - - @property - def n_shots(self) -> list[int]: - return [0] - - @property - def task_group_name(self) -> str: - return "region-reasoner" - - @property - def description(self) -> str: - return ( - "RegionReasoner multi-turn region grounding benchmark (RefCOCOg). " - "Requires REGION_REASONER_DIR on cluster." - ) - - @property - def primary_metric(self) -> str: - return "gIoU" - - @property - def hf_models(self) -> list[str]: - return ["Ricky06662/TaskRouter-1.5B", "facebook/sam2-hiera-large"] - - @property - def hf_dataset_files(self) -> list[dict]: - return [ - { - "repo_id": "lmsdss/regionreasoner_test_data", - "patterns": [ - "raw/refcocog_multi_turn.json", - "raw/refcocog_test_multi_bbox_images/*", - ], - } - ] diff --git a/oellm/contrib/region_reasoner/README.md b/oellm/contrib/regiondial_bench/README.md similarity index 65% rename from oellm/contrib/region_reasoner/README.md rename to oellm/contrib/regiondial_bench/README.md index b9cc90c4..c25c2e72 100644 --- a/oellm/contrib/region_reasoner/README.md +++ b/oellm/contrib/regiondial_bench/README.md @@ -1,9 +1,16 @@ -# RegionReasoner Benchmark +# RegionDial-Bench -Multi-turn region grounding benchmark on RefCOCOg. Evaluates a model's ability -to locate and segment objects described in multi-turn conversations. +Multi-round region grounding benchmark on RefCOCOg and RefCOCO+ +(Sun et al., ICLR 2026). Evaluates a model's ability to locate and segment +objects described in multi-turn conversations, measuring robustness to error +accumulation across dialogue turns. -**Metrics:** gIoU, cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9 +**Splits:** +- RefCOCOg Multi-turn — 1,580 images, 4,405 turns +- RefCOCO+ Multi-turn — 715 images, 2,355 turns + +**Metrics:** gIoU (primary), cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9, +plus per-round breakdown (R1–R7) for gIoU and bbox_AP. --- @@ -31,7 +38,7 @@ my-cluster: ... HF_HOME: "/path/to/large/filesystem/huggingface" # must have ~30 GB free REGION_REASONER_DIR: "/path/to/RegionReasoner" - REGION_REASONER_NUM_GPUS: "4" # optional, default: 4 + GPUS_PER_NODE: 4 # controls both SLURM --gres and shard count ``` > **`HF_HOME`** must point to a filesystem with at least **30 GB** of free @@ -70,8 +77,10 @@ not need internet access): |---|---|---| | TaskRouter-1.5B | `Ricky06662/TaskRouter-1.5B` | ~3 GB | | SAM2 | `facebook/sam2-hiera-large` | ~1 GB | -| Test JSON | `lmsdss/regionreasoner_test_data` `raw/refcocog_multi_turn.json` | ~26 GB | -| Test images | `lmsdss/regionreasoner_test_data` `raw/refcocog_test_multi_bbox_images/*` | ~1 GB | +| RefCOCOg test JSON | `lmsdss/regionreasoner_test_data` `raw/refcocog_multi_turn.json` | ~26 GB | +| RefCOCOg test images | `lmsdss/regionreasoner_test_data` `raw/refcocog_test_multi_bbox_images/*` | ~200 MB (1 580 images) | +| RefCOCO+ test JSON | `lmsdss/regionreasoner_test_data` `raw/refcocoplus_multi_turn.json` | ~13 GB | +| RefCOCO+ test images | `lmsdss/regionreasoner_test_data` `raw/refcocoplus_test_multi_bbox_images/*` | ~93 MB (715 images) | All assets are cached under `$HF_HOME/hub`. @@ -79,10 +88,25 @@ All assets are cached under `$HF_HOME/hub`. ## Running +Three task groups are available: + +| Task group | Splits | +|---|---| +| `regiondial-bench` | Both (RefCOCOg + RefCOCO+) | +| `regiondial-refcocog` | RefCOCOg only (1,580 images, 4,405 turns) | +| `regiondial-refcocoplus` | RefCOCO+ only (715 images, 2,355 turns) | + ```bash +# Both splits +oellm schedule-eval \ + --models lmsdss/RegionReasoner-7B \ + --task_groups regiondial-bench \ + --venv_path ~/elliot-venv + +# Single split oellm schedule-eval \ --models lmsdss/RegionReasoner-7B \ - --task_groups region-reasoner \ + --task_groups regiondial-refcocog \ --venv_path ~/elliot-venv ``` @@ -94,7 +118,9 @@ oellm collect-results \ --output results.csv ``` -The primary metric in the CSV is **gIoU**. +The primary metric in the CSV is **gIoU**. Per-round metrics (e.g. +`gIoU_R1`, `bbox_AP_R3`) are included when the inference script outputs +a `round` field per sample. --- @@ -107,7 +133,7 @@ model, just pass it to `--models`: ```bash oellm schedule-eval \ --models Qwen/Qwen2.5-VL-7B-Instruct \ - --task_groups region-reasoner \ + --task_groups regiondial-bench \ --venv_path ~/elliot-venv ``` @@ -125,7 +151,7 @@ To evaluate multiple models in one go: ```bash oellm schedule-eval \ --models lmsdss/RegionReasoner-7B Qwen/Qwen2.5-VL-7B-Instruct \ - --task_groups region-reasoner \ + --task_groups regiondial-bench \ --venv_path ~/elliot-venv ``` diff --git a/oellm/contrib/regiondial_bench/__init__.py b/oellm/contrib/regiondial_bench/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/oellm/contrib/regiondial_bench/__init__.py @@ -0,0 +1 @@ + diff --git a/oellm/contrib/region_reasoner/adapter.py b/oellm/contrib/regiondial_bench/adapter.py similarity index 90% rename from oellm/contrib/region_reasoner/adapter.py rename to oellm/contrib/regiondial_bench/adapter.py index b849d708..a257c5b3 100644 --- a/oellm/contrib/region_reasoner/adapter.py +++ b/oellm/contrib/regiondial_bench/adapter.py @@ -1,11 +1,11 @@ -"""RegionReasoner model adapter.""" +"""RegionDial-Bench model adapter.""" from pathlib import Path from oellm.core.base_model_adapter import BaseModelAdapter -class RegionReasonerModelAdapter(BaseModelAdapter): +class RegionDialModelAdapter(BaseModelAdapter): """Translates a model path into eval-engine argument strings.""" def __init__(self, model_path: str) -> None: diff --git a/oellm/contrib/region_reasoner/metrics.py b/oellm/contrib/regiondial_bench/metrics.py similarity index 94% rename from oellm/contrib/region_reasoner/metrics.py rename to oellm/contrib/regiondial_bench/metrics.py index 5b84be45..17234336 100644 --- a/oellm/contrib/region_reasoner/metrics.py +++ b/oellm/contrib/regiondial_bench/metrics.py @@ -2,7 +2,7 @@ :class:`BaseMetric` subclasses that compute region-grounding metrics from per-sample inference results. These are used by ``suite._aggregate_shards()`` -to score any model's predictions. +to score any model's predictions on RegionDial-Bench. Input format ------------ @@ -12,7 +12,8 @@ { "intersection": 12345, "union": 23456, - "bbox_iou": 0.73 + "bbox_iou": 0.73, + "round": 1 } ``predictions`` passed to ``compute()`` are ``list[str]`` — one JSON string @@ -57,7 +58,7 @@ def _mask_iou(sample: dict) -> float: class GIoU(BaseMetric): - """Mean per-sample mask IoU (gIoU as reported in the RegionReasoner paper). + """Mean per-sample mask IoU (gIoU as reported in RegionDial-Bench). Formula: ``mean(intersection_i / union_i)`` over all samples. """ @@ -77,7 +78,7 @@ def compute(self, predictions: list[str], references: list[str]) -> float: class CIoU(BaseMetric): - """Cumulative IoU (cIoU as reported in the RegionReasoner paper). + """Cumulative IoU (cIoU as reported in RegionDial-Bench). Formula: ``sum(all intersections) / sum(all unions)``. """ diff --git a/oellm/contrib/regiondial_bench/suite.py b/oellm/contrib/regiondial_bench/suite.py new file mode 100644 index 00000000..31742385 --- /dev/null +++ b/oellm/contrib/regiondial_bench/suite.py @@ -0,0 +1,399 @@ +"""RegionDial-Bench contrib suite — plugin protocol implementation. + +This module follows the plugin protocol defined in ``oellm/registry.py``. +It is the reference implementation for custom benchmark integration. + +RegionDial-Bench (Sun et al., ICLR 2026) is a multi-round benchmark for +reference-grounded region reasoning, built on RefCOCO+ and RefCOCOg. + +Cluster setup +------------- +The following environment variables must be set in ``clusters.yaml`` (or the +cluster's module/profile system) before using the ``regiondial-bench`` task group: + +``REGION_REASONER_DIR`` + Absolute path to a local clone of the RegionReasoner repository + (https://github.com/lmsdss/RegionReasoner). The eval scripts + ``test/evaluation/evaluation_multi_segmentation.py`` must be present. + +The number of GPUs is read from ``GPUS_PER_NODE`` (set in ``clusters.yaml``), +which also controls the SLURM ``--gres=gpu:`` request. + +Output format +------------- +``run()`` writes a **lmms-eval-compatible JSON** file so that +``oellm.main.collect_results()`` works without modification:: + + { + "model_name_or_path": "", + "results": { + "regiondial_refcocog": { + "gIoU": 0.42, "cIoU": 0.45, "bbox_AP": 0.38, + "pass_rate_0.3": 0.71, "pass_rate_0.5": 0.55, + "pass_rate_0.7": 0.31, "pass_rate_0.9": 0.08, + "gIoU_R1": 0.55, "gIoU_R2": 0.48, ... + } + }, + "configs": { + "regiondial_refcocog": {"num_fewshot": 0} + } + } +""" + +from __future__ import annotations + +import json +import logging +import subprocess +import tempfile +from collections import defaultdict +from pathlib import Path + +logger = logging.getLogger(__name__) + +SUITE_NAME = "regiondial_bench" + +CLUSTER_ENV_VARS = [ + "REGION_REASONER_DIR", +] + +from oellm.contrib.regiondial_bench.task import ( # noqa: E402 + RegionDialRefCOCOgTask, + RegionDialRefCOCOplusTask, +) + +_refcocog = RegionDialRefCOCOgTask.to_task_groups_dict() +_refcocoplus = RegionDialRefCOCOplusTask.to_task_groups_dict() + +_all_name = "regiondial-bench" +_all_metrics = { + **_refcocog.get("task_metrics", {}), + **_refcocoplus.get("task_metrics", {}), +} + +_group_kwargs = {"suite": SUITE_NAME, "n_shots": [0]} + +TASK_GROUPS: dict = { + "task_metrics": _all_metrics, + "task_groups": { + _all_name: { + **_group_kwargs, + "description": ( + "RegionDial-Bench: both splits — RefCOCOg + RefCOCO+ " + "(Sun et al., ICLR 2026)." + ), + "tasks": ( + _refcocog["task_groups"][_all_name]["tasks"] + + _refcocoplus["task_groups"][_all_name]["tasks"] + ), + }, + "regiondial-refcocog": { + **_group_kwargs, + "description": ( + "RegionDial-Bench: RefCOCOg Multi-turn only (1,580 images, 4,405 turns)." + ), + "tasks": _refcocog["task_groups"][_all_name]["tasks"], + }, + "regiondial-refcocoplus": { + **_group_kwargs, + "description": ( + "RegionDial-Bench: RefCOCO+ Multi-turn only (715 images, 2,355 turns)." + ), + "tasks": _refcocoplus["task_groups"][_all_name]["tasks"], + }, + }, +} + +_TASK_JSON_FILES: dict[str, str] = { + "regiondial_refcocog": "refcocog_multi_turn.json", + "regiondial_refcocoplus": "refcocoplus_multi_turn.json", +} + + +def detect_model_flags(model_path: str) -> str | None: + """Delegate to RegionDialModelAdapter.to_contrib_flags().""" + from oellm.contrib.regiondial_bench.adapter import RegionDialModelAdapter + + return RegionDialModelAdapter(model_path).to_contrib_flags() + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Execute the RegionDial-Bench evaluation and write results to *output_path*. + + This function: + + 1. Resolves the correct test JSON for the requested split (RefCOCOg or + RefCOCO+) based on the *task* name. + 2. Runs ``evaluation_multi_segmentation.py`` in parallel GPU shards. + 3. Computes aggregate and per-round (R1–R7) metrics using + :mod:`oellm.contrib.regiondial_bench.metrics`. + 4. Writes a lmms-eval-compatible JSON to *output_path*. + + Args: + model_path: Path or HF repo ID of the model checkpoint. + task: Task name (``"regiondial_refcocog"`` or ``"regiondial_refcocoplus"``). + n_shot: Number of few-shot examples (always 0 for this benchmark). + output_path: Where to write the results JSON. + model_flags: Model type string (e.g. ``"vision_reasoner"``). + env: Environment variables dict (from ``os.environ``). + """ + rr_dir = env.get("REGION_REASONER_DIR", "") + num_gpus = int(env.get("GPUS_PER_NODE", "1")) + model_type = model_flags or "vision_reasoner" + + if not rr_dir: + raise RuntimeError( + "REGION_REASONER_DIR must be set. Add it to clusters.yaml for this cluster." + ) + + json_filename = _TASK_JSON_FILES.get(task) + if not json_filename: + raise ValueError( + f"Unknown task {task!r}. Expected one of: {list(_TASK_JSON_FILES)}" + ) + + test_json = _resolve_test_json(task, json_filename, env) + + inference_script = ( + Path(rr_dir) / "test" / "evaluation" / "evaluation_multi_segmentation.py" + ) + if not inference_script.exists(): + raise FileNotFoundError( + f"RegionReasoner inference script not found: {inference_script}\n" + f"Check that REGION_REASONER_DIR={rr_dir!r} points to a valid clone." + ) + + with tempfile.TemporaryDirectory(prefix="rr_shards_") as tmp_dir: + shard_paths = _stream_preshard(test_json, tmp_dir, num_gpus) + + procs = [] + for idx in range(num_gpus): + shard_env = dict(env) + shard_env["CUDA_VISIBLE_DEVICES"] = str(idx) + cmd = [ + "python", + str(inference_script), + "--model_path", + model_path, + "--model", + model_type, + "--test_data_path", + shard_paths[idx], + "--output_path", + tmp_dir, + "--vis_output_path", + str(Path(tmp_dir) / f"vis_{idx}"), + "--idx", + str(idx), + "--num_parts", + "1", + "--batch_size", + "2", + "--task_router_model_path", + "Ricky06662/TaskRouter-1.5B", + ] + logger.info("Starting shard %d/%d: %s", idx + 1, num_gpus, " ".join(cmd)) + proc = subprocess.Popen(cmd, env=shard_env, cwd=str(Path(test_json).parent)) + procs.append(proc) + + for idx, proc in enumerate(procs): + ret = proc.wait() + if ret != 0: + raise RuntimeError( + f"RegionDial-Bench inference shard {idx} exited with code {ret}" + ) + + logger.info("All %d shards completed. Computing metrics.", num_gpus) + + metrics = _aggregate_shards(tmp_dir) + + result_json = { + "model_name_or_path": model_path, + "results": {task: metrics}, + "configs": {task: {"num_fewshot": n_shot}}, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(result_json, f, indent=2) + logger.info("Results written to %s", output_path) + + +def _stream_preshard(json_path: str, out_dir: str, num_shards: int) -> list[str]: + """Split a large JSON array into *num_shards* files using streaming. + + Uses ``ijson`` to iterate over the top-level array without loading the + entire file into memory. Items are distributed round-robin. + + Returns a list of shard file paths. + """ + import ijson + + shard_files = [] + shard_counts = [0] * num_shards + for idx in range(num_shards): + p = str(Path(out_dir) / f"shard_{idx}.json") + shard_files.append(open(p, "w")) # noqa: SIM115 + shard_files[-1].write("[\n") + + logger.info("Streaming pre-shard of %s into %d files", json_path, num_shards) + + with open(json_path, "rb") as f: + for i, item in enumerate(ijson.items(f, "item")): + shard_idx = i % num_shards + if shard_counts[shard_idx] > 0: + shard_files[shard_idx].write(",\n") + json.dump(item, shard_files[shard_idx]) + shard_counts[shard_idx] += 1 + + shard_paths = [] + for idx in range(num_shards): + shard_files[idx].write("\n]") + shard_files[idx].close() + shard_paths.append(str(Path(out_dir) / f"shard_{idx}.json")) + logger.info("Shard %d: %d samples", idx, shard_counts[idx]) + + logger.info("Pre-sharding complete: %d total samples", sum(shard_counts)) + return shard_paths + + +def _resolve_test_json(task: str, json_filename: str, env: dict[str, str]) -> str: + """Resolve the path to the test JSON for the given split. + + Uses env-var overrides if present, otherwise auto-downloads from HF Hub. + """ + split_key = task.replace("regiondial_", "").upper() + env_var = f"REGION_REASONER_TEST_JSON_{split_key}" + override = env.get(env_var) + if override: + return override + + # Backward compat: single env var that contains the filename + legacy = env.get("REGION_REASONER_TEST_JSON") + if legacy and json_filename in legacy: + return legacy + + from huggingface_hub import snapshot_download + + split_prefix = json_filename.replace("_multi_turn.json", "") + local_dir = snapshot_download( + repo_id="lmsdss/regionreasoner_test_data", + repo_type="dataset", + allow_patterns=[ + f"raw/{json_filename}", + f"raw/{split_prefix}_test_multi_bbox_images/*", + ], + cache_dir=Path(env["HF_HOME"]) / "hub" if "HF_HOME" in env else None, + ) + return str(Path(local_dir) / "raw" / json_filename) + + +def _aggregate_shards(shard_dir: str) -> dict[str, float]: + """Read per-shard output files and compute all metrics. + + Each shard file contains a list of per-sample dicts with pre-computed + ``intersection``, ``union``, ``bbox_iou``, and ``round`` fields written + by the upstream ``evaluation_multi_segmentation.py`` script. + + Computes: + - Aggregate metrics across all rounds: gIoU, cIoU, bbox_AP, pass_rate_* + - Per-round metrics (R1–R7): gIoU_R1..R7, bbox_AP_R1..R7 + + Returns a flat dict of ``{metric_name: value}``. + """ + from oellm.contrib.regiondial_bench.metrics import ( + BboxAP, + CIoU, + GIoU, + PassRate, + ) + + shard_files = sorted(Path(shard_dir).glob("output_*.json")) + if not shard_files: + raise RuntimeError( + f"No shard output files found in {shard_dir!r}. " + "The inference script may have failed silently." + ) + + all_samples: list[dict] = [] + for shard_file in shard_files: + with open(shard_file) as f: + shard_data = json.load(f) + all_samples.extend(shard_data) + + if not all_samples: + raise RuntimeError( + "No samples found across shard files. " + "The inference script produced empty output." + ) + logger.info( + "Aggregating %d samples from %d shards", len(all_samples), len(shard_files) + ) + + samples = [json.dumps(s) for s in all_samples] + empty_refs = [""] * len(samples) + + aggregate_metrics = [ + GIoU(), + CIoU(), + BboxAP(), + PassRate(0.3), + PassRate(0.5), + PassRate(0.7), + PassRate(0.9), + ] + + metrics: dict[str, float] = {} + for m in aggregate_metrics: + val = m.compute(samples, empty_refs) + metrics[m.name] = val + logger.debug("%s = %.4f", m.name, val) + + rounds_map: dict[int, list[str]] = defaultdict(list) + for sample_dict, sample_str in zip(all_samples, samples, strict=True): + rnd = sample_dict.get("round") + if rnd is not None: + rounds_map[int(rnd)].append(sample_str) + + if rounds_map: + per_round_metrics = [GIoU(), BboxAP()] + for rnd in sorted(rounds_map): + rnd_samples = rounds_map[rnd] + rnd_refs = [""] * len(rnd_samples) + for m in per_round_metrics: + val = m.compute(rnd_samples, rnd_refs) + metrics[f"{m.name}_R{rnd}"] = val + logger.debug("%s_R%d = %.4f", m.name, rnd, val) + else: + logger.warning( + "No 'round' field found in samples — skipping per-round breakdown. " + "Per-round metrics (R1–R7) require the inference script to output " + "a 'round' field in each sample." + ) + + return metrics + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Try to parse *data* as a RegionDial-Bench output JSON. + + Returns ``(model_id, task_name, n_shot, {metric: value})`` if the JSON + matches this suite's format, otherwise ``None``. + + Detection heuristic: the ``results`` dict contains a key that starts with + ``"regiondial_"`` and the value dict contains ``"gIoU"``. + """ + results = data.get("results", {}) + for task_name, task_results in results.items(): + if task_name.startswith("regiondial_") and "gIoU" in task_results: + model_id = data.get("model_name_or_path") or data.get("model_name", "unknown") + n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) + return model_id, task_name, int(n_shot), task_results + return None diff --git a/oellm/contrib/regiondial_bench/task.py b/oellm/contrib/regiondial_bench/task.py new file mode 100644 index 00000000..c64db668 --- /dev/null +++ b/oellm/contrib/regiondial_bench/task.py @@ -0,0 +1,107 @@ +"""RegionDial-Bench task definitions. + +Two splits from the RegionDial-Bench benchmark (Sun et al., ICLR 2026): +- RefCOCOg Multi-turn (1,580 images, 4,405 turns) +- RefCOCO+ Multi-turn (715 images, 2,355 turns) +""" + +from oellm.core.base_task import BaseTask + +_TASK_GROUP_ALL = "regiondial-bench" +_SUITE = "regiondial_bench" +_HF_MODELS = ["Ricky06662/TaskRouter-1.5B", "facebook/sam2-hiera-large"] +_HF_REPO = "lmsdss/regionreasoner_test_data" + + +class RegionDialRefCOCOgTask(BaseTask): + """RegionDial-Bench — RefCOCOg Multi-turn split.""" + + @property + def name(self) -> str: + return "regiondial_refcocog" + + @property + def suite(self) -> str: + return _SUITE + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def task_group_name(self) -> str: + return _TASK_GROUP_ALL + + @property + def description(self) -> str: + return ( + "RegionDial-Bench RefCOCOg Multi-turn split (1,580 images, 4,405 turns). " + "Requires REGION_REASONER_DIR on cluster." + ) + + @property + def primary_metric(self) -> str: + return "gIoU" + + @property + def hf_models(self) -> list[str]: + return _HF_MODELS + + @property + def hf_dataset_files(self) -> list[dict]: + return [ + { + "repo_id": _HF_REPO, + "patterns": [ + "raw/refcocog_multi_turn.json", + "raw/refcocog_test_multi_bbox_images/*", + ], + } + ] + + +class RegionDialRefCOCOplusTask(BaseTask): + """RegionDial-Bench — RefCOCO+ Multi-turn split.""" + + @property + def name(self) -> str: + return "regiondial_refcocoplus" + + @property + def suite(self) -> str: + return _SUITE + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def task_group_name(self) -> str: + return _TASK_GROUP_ALL + + @property + def description(self) -> str: + return ( + "RegionDial-Bench RefCOCO+ Multi-turn split (715 images, 2,355 turns). " + "Requires REGION_REASONER_DIR on cluster." + ) + + @property + def primary_metric(self) -> str: + return "gIoU" + + @property + def hf_models(self) -> list[str]: + return _HF_MODELS + + @property + def hf_dataset_files(self) -> list[dict]: + return [ + { + "repo_id": _HF_REPO, + "patterns": [ + "raw/refcocoplus_multi_turn.json", + "raw/refcocoplus_test_multi_bbox_images/*", + ], + } + ] diff --git a/oellm/core/base_task.py b/oellm/core/base_task.py index f2ccd969..2d779e44 100644 --- a/oellm/core/base_task.py +++ b/oellm/core/base_task.py @@ -49,7 +49,7 @@ def suite(self) -> str: """Evaluation suite identifier. One of: ``lm_eval``, ``lighteval``, ``lmms_eval``, or a contrib - ``SUITE_NAME`` (e.g. ``"region_reasoner"``). + ``SUITE_NAME`` (e.g. ``"regiondial_bench"``). """ @property diff --git a/oellm/main.py b/oellm/main.py index 9f02da02..a027e92c 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -288,10 +288,12 @@ def schedule_evals( f"evals to schedule: {len(df)}." ) - evals_dir = ( - Path(os.environ["EVAL_OUTPUT_DIR"]) - / f"{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}" - ) + # Build a descriptive directory name: {models}_{task_groups}_{timestamp} + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + model_names = "+".join(m.split("/")[-1].lower() for m in (models or [])) + group_label = "+".join(g.lower() for g in (group_names or [])) + parts = [p for p in [model_names, group_label, timestamp] if p] + evals_dir = Path(os.environ["EVAL_OUTPUT_DIR"]) / "_".join(parts) evals_dir.mkdir(parents=True, exist_ok=True) slurm_logs_dir = evals_dir / "slurm_logs" diff --git a/oellm/registry.py b/oellm/registry.py index 8c4c1ad7..bbbba335 100644 --- a/oellm/registry.py +++ b/oellm/registry.py @@ -10,7 +10,7 @@ Required ~~~~~~~~ ``SUITE_NAME: str`` - Identifier used in the ``eval_suite`` CSV column, e.g. ``"region_reasoner"``. + Identifier used in the ``eval_suite`` CSV column, e.g. ``"regiondial_bench"``. ``TASK_GROUPS: dict`` Task-group definitions in ``task-groups.yaml`` format. Expected keys: diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 7fbe5fd3..9a91f803 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -2,7 +2,7 @@ #SBATCH --job-name=oellm-eval #SBATCH --time={time_limit} #SBATCH --gres=gpu:$GPUS_PER_NODE -#SBATCH --mem=$MEM_PER_NODE +#SBATCH --mem=0 #SBATCH --output={log_dir}/%x-%A-%a.out #SBATCH --partition=$PARTITION #SBATCH --account=$ACCOUNT @@ -102,7 +102,7 @@ do GPU_DEVICES=$(seq -s, 0 $(($GPUS_PER_NODE - 1))) - # Strip optional model-flags suffix ("region_reasoner:vision_reasoner" → "region_reasoner") + # Strip optional model-flags suffix ("regiondial_bench:vision_reasoner" → "regiondial_bench") suite_normalized=$(echo "${{eval_suite%%:*}}" | tr '[:upper:]' '[:lower:]') # Helper function to run Python commands in the appropriate environment diff --git a/oellm/task_groups.py b/oellm/task_groups.py index b51885f2..e9f2c236 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -236,17 +236,24 @@ def _collect_hf_dataset_files(group_names: Iterable[str]) -> list[dict]: """Return deduplicated HF dataset file specs declared in task ``hf_dataset_files`` fields.""" parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) - file_specs: list[dict] = [] - seen: set[str] = set() + # Merge patterns from all tasks that share the same repo_id so that + # a single snapshot_download fetches everything needed. + merged: dict[str, list[str]] = {} for t, _ in _iter_all_tasks(parsed): for spec in t.hf_dataset_files or []: repo_id = spec.get("repo_id", "") - if repo_id and repo_id not in seen: - seen.add(repo_id) - file_specs.append(spec) + if not repo_id: + continue + patterns = spec.get("patterns") or [] + if repo_id not in merged: + merged[repo_id] = list(patterns) + else: + for p in patterns: + if p not in merged[repo_id]: + merged[repo_id].append(p) - return file_specs + return [{"repo_id": rid, "patterns": pats} for rid, pats in merged.items()] def _build_task_dataset_map() -> dict[str, list[DatasetSpec]]: diff --git a/pyproject.toml b/pyproject.toml index 46234a54..dd5be106 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "rich", "huggingface_hub", "pyyaml", + "ijson", ] [project.optional-dependencies] diff --git a/tests/test_region_reasoner.py b/tests/test_regiondial_bench.py similarity index 59% rename from tests/test_region_reasoner.py rename to tests/test_regiondial_bench.py index fb129727..425cd966 100644 --- a/tests/test_region_reasoner.py +++ b/tests/test_regiondial_bench.py @@ -1,4 +1,4 @@ -"""Tests for the RegionReasoner contrib benchmark integration.""" +"""Tests for the RegionDial-Bench contrib benchmark integration.""" import json import os @@ -17,9 +17,10 @@ get_all_task_group_names, ) -RR_TASK_GROUP = "region-reasoner" -RR_TASK_NAME = "regionreasoner_refcocog" -RR_TEST_DATA_REPO = "lmsdss/regionreasoner_test_data" +RD_TASK_GROUP = "regiondial-bench" +RD_TASK_REFCOCOG = "regiondial_refcocog" +RD_TASK_REFCOCOPLUS = "regiondial_refcocoplus" +RD_TEST_DATA_REPO = "lmsdss/regionreasoner_test_data" # --------------------------------------------------------------------------- @@ -27,88 +28,87 @@ # --------------------------------------------------------------------------- -class TestRegionReasonerTaskGroup: +class TestRegionDialBenchTaskGroup: def test_task_group_present_in_all_names(self): - assert RR_TASK_GROUP in get_all_task_group_names() + assert RD_TASK_GROUP in get_all_task_group_names() - def test_task_group_expands_to_correct_task(self): - results = _expand_task_groups([RR_TASK_GROUP]) + def test_task_group_expands_to_both_splits(self): + results = _expand_task_groups([RD_TASK_GROUP]) task_names = {r.task for r in results} - assert RR_TASK_NAME in task_names + assert RD_TASK_REFCOCOG in task_names + assert RD_TASK_REFCOCOPLUS in task_names - def test_task_group_suite_is_region_reasoner(self): - results = _expand_task_groups([RR_TASK_GROUP]) + def test_task_group_suite_is_regiondial_bench(self): + results = _expand_task_groups([RD_TASK_GROUP]) for r in results: - assert r.suite == "region_reasoner", ( - f"Expected suite 'region_reasoner', got '{r.suite}'" + assert r.suite == "regiondial_bench", ( + f"Expected suite 'regiondial_bench', got '{r.suite}'" ) def test_task_group_n_shot_is_zero(self): - results = _expand_task_groups([RR_TASK_GROUP]) + results = _expand_task_groups([RD_TASK_GROUP]) for r in results: assert r.n_shot == 0 def test_no_dataset_pre_download(self): - # No HuggingFace dataset (load_dataset-style) is declared for this task group. - specs = _collect_dataset_specs([RR_TASK_GROUP]) + specs = _collect_dataset_specs([RD_TASK_GROUP]) assert specs == [] def test_hf_dataset_files_declared(self): - import oellm.contrib.region_reasoner.suite as s + import oellm.contrib.regiondial_bench.suite as s - tasks = s.TASK_GROUPS["task_groups"][RR_TASK_GROUP]["tasks"] + tasks = s.TASK_GROUPS["task_groups"][RD_TASK_GROUP]["tasks"] repo_ids = [ spec["repo_id"] for task in tasks for spec in task.get("hf_dataset_files", []) ] - assert RR_TEST_DATA_REPO in repo_ids + assert RD_TEST_DATA_REPO in repo_ids - def test_collect_hf_dataset_files_returns_correct_spec(self): - specs = _collect_hf_dataset_files([RR_TASK_GROUP]) - assert len(specs) == 1 - assert specs[0]["repo_id"] == RR_TEST_DATA_REPO - assert "raw/refcocog_multi_turn.json" in specs[0]["patterns"] + def test_collect_hf_dataset_files_returns_correct_repo(self): + # _collect_hf_dataset_files deduplicates by repo_id, so both splits + # (same repo) produce a single spec entry. + specs = _collect_hf_dataset_files([RD_TASK_GROUP]) + assert len(specs) >= 1 + assert specs[0]["repo_id"] == RD_TEST_DATA_REPO - def test_task_groups_generated_from_task_class(self): - # TASK_GROUPS must be generated from RegionReasonerTask, not hardcoded. - from oellm.contrib.region_reasoner.task import RegionReasonerTask + def test_task_groups_contains_both_task_metrics(self): + import oellm.contrib.regiondial_bench.suite as s - generated = RegionReasonerTask.to_task_groups_dict() - import oellm.contrib.region_reasoner.suite as s - - assert s.TASK_GROUPS == generated + assert RD_TASK_REFCOCOG in s.TASK_GROUPS["task_metrics"] + assert RD_TASK_REFCOCOPLUS in s.TASK_GROUPS["task_metrics"] # --------------------------------------------------------------------------- -# BaseTask subclass +# BaseTask subclasses # --------------------------------------------------------------------------- -class TestRegionReasonerTask: +class TestRegionDialRefCOCOgTask: @pytest.fixture def task(self): - from oellm.contrib.region_reasoner.task import RegionReasonerTask + from oellm.contrib.regiondial_bench.task import RegionDialRefCOCOgTask - return RegionReasonerTask() + return RegionDialRefCOCOgTask() def test_is_base_task_instance(self, task): assert isinstance(task, BaseTask) def test_name(self, task): - assert task.name == RR_TASK_NAME + assert task.name == RD_TASK_REFCOCOG def test_suite(self, task): - assert task.suite == "region_reasoner" + assert task.suite == "regiondial_bench" def test_n_shots(self, task): assert task.n_shots == [0] def test_dataset_specs_empty(self, task): - # Data is accessed via hf_dataset_files (snapshot_download), not load_dataset. assert task.dataset_specs == [] def test_hf_dataset_files(self, task): repo_ids = [f["repo_id"] for f in task.hf_dataset_files] - assert RR_TEST_DATA_REPO in repo_ids + assert RD_TEST_DATA_REPO in repo_ids + patterns = task.hf_dataset_files[0]["patterns"] + assert "raw/refcocog_multi_turn.json" in patterns def test_hf_models(self, task): assert "Ricky06662/TaskRouter-1.5B" in task.hf_models @@ -118,7 +118,7 @@ def test_primary_metric(self, task): assert task.primary_metric == "gIoU" def test_task_group_name(self, task): - assert task.task_group_name == RR_TASK_GROUP + assert task.task_group_name == RD_TASK_GROUP def test_engine_task_name_defaults_to_name(self, task): assert task.engine_task_name == task.name @@ -126,10 +126,55 @@ def test_engine_task_name_defaults_to_name(self, task): def test_to_task_groups_dict_structure(self, task): d = task.to_task_groups_dict() assert "task_metrics" in d - assert d["task_metrics"][RR_TASK_NAME] == "gIoU" - assert RR_TASK_GROUP in d["task_groups"] - tasks = d["task_groups"][RR_TASK_GROUP]["tasks"] - assert any(t["task"] == RR_TASK_NAME for t in tasks) + assert d["task_metrics"][RD_TASK_REFCOCOG] == "gIoU" + assert RD_TASK_GROUP in d["task_groups"] + tasks = d["task_groups"][RD_TASK_GROUP]["tasks"] + assert any(t["task"] == RD_TASK_REFCOCOG for t in tasks) + + +class TestRegionDialRefCOCOplusTask: + @pytest.fixture + def task(self): + from oellm.contrib.regiondial_bench.task import RegionDialRefCOCOplusTask + + return RegionDialRefCOCOplusTask() + + def test_is_base_task_instance(self, task): + assert isinstance(task, BaseTask) + + def test_name(self, task): + assert task.name == RD_TASK_REFCOCOPLUS + + def test_suite(self, task): + assert task.suite == "regiondial_bench" + + def test_n_shots(self, task): + assert task.n_shots == [0] + + def test_dataset_specs_empty(self, task): + assert task.dataset_specs == [] + + def test_hf_dataset_files(self, task): + repo_ids = [f["repo_id"] for f in task.hf_dataset_files] + assert RD_TEST_DATA_REPO in repo_ids + patterns = task.hf_dataset_files[0]["patterns"] + assert "raw/refcocoplus_multi_turn.json" in patterns + + def test_hf_models(self, task): + assert "Ricky06662/TaskRouter-1.5B" in task.hf_models + assert "facebook/sam2-hiera-large" in task.hf_models + + def test_primary_metric(self, task): + assert task.primary_metric == "gIoU" + + def test_task_group_name(self, task): + assert task.task_group_name == RD_TASK_GROUP + + def test_to_task_groups_dict_structure(self, task): + d = task.to_task_groups_dict() + assert d["task_metrics"][RD_TASK_REFCOCOPLUS] == "gIoU" + tasks = d["task_groups"][RD_TASK_GROUP]["tasks"] + assert any(t["task"] == RD_TASK_REFCOCOPLUS for t in tasks) # --------------------------------------------------------------------------- @@ -137,21 +182,24 @@ def test_to_task_groups_dict_structure(self, task): # --------------------------------------------------------------------------- -def _sample(intersection: int, union: int, bbox_iou: float = 0.0) -> str: +def _sample( + intersection: int, union: int, bbox_iou: float = 0.0, round: int | None = None +) -> str: """Helper: JSON-serialise a sample dict for metric inputs.""" - return json.dumps( - { - "intersection": intersection, - "union": union, - "bbox_iou": bbox_iou, - } - ) + d = { + "intersection": intersection, + "union": union, + "bbox_iou": bbox_iou, + } + if round is not None: + d["round"] = round + return json.dumps(d) class TestGIoU: @pytest.fixture def metric(self): - from oellm.contrib.region_reasoner.metrics import GIoU + from oellm.contrib.regiondial_bench.metrics import GIoU return GIoU() @@ -190,7 +238,7 @@ def test_null_sample(self, metric): class TestCIoU: @pytest.fixture def metric(self): - from oellm.contrib.region_reasoner.metrics import CIoU + from oellm.contrib.regiondial_bench.metrics import CIoU return CIoU() @@ -209,11 +257,9 @@ def test_zero_overlap(self, metric): assert metric.compute([s], [""]) == pytest.approx(0.0) def test_cumulative_formula_differs_from_giou(self, metric): - from oellm.contrib.region_reasoner.metrics import GIoU + from oellm.contrib.regiondial_bench.metrics import GIoU giou = GIoU() - # Sample 1: IoU = 100/100 = 1.0 - # Sample 2: IoU = 50/200 = 0.25 s1 = _sample(100, 100) s2 = _sample(50, 200) preds = [s1, s2] @@ -231,7 +277,7 @@ def test_empty_input(self, metric): class TestBboxAP: @pytest.fixture def metric(self): - from oellm.contrib.region_reasoner.metrics import BboxAP + from oellm.contrib.regiondial_bench.metrics import BboxAP return BboxAP() @@ -264,32 +310,32 @@ def threshold(self, request): return request.param def test_name_includes_threshold(self, threshold): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate pr = PassRate(threshold) assert pr.name == f"pass_rate_{threshold}" def test_is_base_metric(self): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate assert isinstance(PassRate(0.5), BaseMetric) def test_all_pass(self): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate s = _sample(100, 100) pr = PassRate(0.5) assert pr.compute([s, s], ["", ""]) == pytest.approx(1.0) def test_none_pass(self): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate s = _sample(0, 100) pr = PassRate(0.3) assert pr.compute([s], [""]) == pytest.approx(0.0) def test_half_pass(self): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate perfect = _sample(100, 100) zero = _sample(0, 100) @@ -298,7 +344,7 @@ def test_half_pass(self): assert score == pytest.approx(0.5) def test_invalid_threshold_raises(self): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate with pytest.raises(ValueError): PassRate(0.0) @@ -308,7 +354,7 @@ def test_invalid_threshold_raises(self): PassRate(-0.1) def test_empty_input(self): - from oellm.contrib.region_reasoner.metrics import PassRate + from oellm.contrib.regiondial_bench.metrics import PassRate assert PassRate(0.5).compute([], []) == pytest.approx(0.0) @@ -321,12 +367,12 @@ def test_empty_input(self): class TestSuiteProtocol: @pytest.fixture def suite(self): - import oellm.contrib.region_reasoner.suite as s + import oellm.contrib.regiondial_bench.suite as s return s def test_suite_name(self, suite): - assert suite.SUITE_NAME == "region_reasoner" + assert suite.SUITE_NAME == "regiondial_bench" def test_cluster_env_vars_declared(self, suite): assert "REGION_REASONER_DIR" in suite.CLUSTER_ENV_VARS @@ -335,8 +381,15 @@ def test_task_groups_structure(self, suite): tg = suite.TASK_GROUPS assert "task_metrics" in tg assert "task_groups" in tg - assert RR_TASK_NAME in tg["task_metrics"] - assert RR_TASK_GROUP in tg["task_groups"] + assert RD_TASK_REFCOCOG in tg["task_metrics"] + assert RD_TASK_REFCOCOPLUS in tg["task_metrics"] + assert RD_TASK_GROUP in tg["task_groups"] + + def test_task_groups_has_both_tasks(self, suite): + tasks = suite.TASK_GROUPS["task_groups"][RD_TASK_GROUP]["tasks"] + task_names = {t["task"] for t in tasks} + assert RD_TASK_REFCOCOG in task_names + assert RD_TASK_REFCOCOPLUS in task_names def test_detect_model_flags_region_reasoner_model(self, suite): assert suite.detect_model_flags("lmsdss/RegionReasoner-7B") == "vision_reasoner" @@ -350,28 +403,45 @@ def test_detect_model_flags_qwen1_model(self, suite): def test_detect_model_flags_unknown_defaults_to_vision_reasoner(self, suite): assert suite.detect_model_flags("some/unknown-model") == "vision_reasoner" - def test_parse_results_valid_json(self, suite): + def test_parse_results_refcocog_json(self, suite): data = { "model_name_or_path": "/path/to/model", "results": { - RR_TASK_NAME: { + RD_TASK_REFCOCOG: { "gIoU": 0.42, "cIoU": 0.45, "bbox_AP": 0.38, } }, - "configs": {RR_TASK_NAME: {"num_fewshot": 0}}, + "configs": {RD_TASK_REFCOCOG: {"num_fewshot": 0}}, } result = suite.parse_results(data) assert result is not None model_id, task_name, n_shot, metrics = result assert model_id == "/path/to/model" - assert task_name == RR_TASK_NAME + assert task_name == RD_TASK_REFCOCOG assert n_shot == 0 assert metrics["gIoU"] == pytest.approx(0.42) + def test_parse_results_refcocoplus_json(self, suite): + data = { + "model_name_or_path": "/path/to/model", + "results": { + RD_TASK_REFCOCOPLUS: { + "gIoU": 0.55, + "cIoU": 0.50, + "bbox_AP": 0.48, + } + }, + "configs": {RD_TASK_REFCOCOPLUS: {"num_fewshot": 0}}, + } + result = suite.parse_results(data) + assert result is not None + _, task_name, _, metrics = result + assert task_name == RD_TASK_REFCOCOPLUS + assert metrics["gIoU"] == pytest.approx(0.55) + def test_parse_results_non_matching_json_returns_none(self, suite): - # lm-eval output format — should not be parsed by this suite data = { "model_name": "some_model", "results": {"mmlu": {"acc,none": 0.55}}, @@ -388,13 +458,13 @@ def test_parse_results_empty_results_returns_none(self, suite): # --------------------------------------------------------------------------- -class TestRegionReasonerModelAdapter: +class TestRegionDialModelAdapter: @pytest.fixture def adapter_cls(self): - from oellm.contrib.region_reasoner.adapter import RegionReasonerModelAdapter + from oellm.contrib.regiondial_bench.adapter import RegionDialModelAdapter from oellm.core.base_model_adapter import BaseModelAdapter - return RegionReasonerModelAdapter, BaseModelAdapter + return RegionDialModelAdapter, BaseModelAdapter def test_is_base_model_adapter(self, adapter_cls): cls, base = adapter_cls @@ -417,7 +487,7 @@ def test_contrib_flags_unknown_defaults_to_vision_reasoner(self, adapter_cls): assert cls("some/unknown-model").to_contrib_flags() == "vision_reasoner" def test_detect_model_flags_delegates_to_adapter(self): - import oellm.contrib.region_reasoner.suite as s + import oellm.contrib.regiondial_bench.suite as s assert s.detect_model_flags("lmsdss/RegionReasoner-7B") == "vision_reasoner" assert s.detect_model_flags("Qwen/Qwen2.5-VL-7B") == "qwen2" @@ -428,7 +498,7 @@ def test_detect_model_flags_delegates_to_adapter(self): # --------------------------------------------------------------------------- -class TestRegionReasonerSchedule: +class TestRegionDialSchedule: def test_schedule_evals_dry_run(self, tmp_path): from oellm.main import schedule_evals @@ -439,7 +509,7 @@ def test_schedule_evals_dry_run(self, tmp_path): ): schedule_evals( models="lmsdss/RegionReasoner-7B", - task_groups=RR_TASK_GROUP, + task_groups=RD_TASK_GROUP, skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, @@ -448,10 +518,9 @@ def test_schedule_evals_dry_run(self, tmp_path): sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) assert len(sbatch_files) == 1 sbatch_content = sbatch_files[0].read_text() - # The contrib catch-all case must be present assert "oellm.contrib.dispatch" in sbatch_content - def test_jobs_csv_has_region_reasoner_suite(self, tmp_path): + def test_jobs_csv_has_regiondial_bench_suite(self, tmp_path): import pandas as pd from oellm.main import schedule_evals @@ -463,7 +532,7 @@ def test_jobs_csv_has_region_reasoner_suite(self, tmp_path): ): schedule_evals( models="lmsdss/RegionReasoner-7B", - task_groups=RR_TASK_GROUP, + task_groups=RD_TASK_GROUP, skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, @@ -472,9 +541,8 @@ def test_jobs_csv_has_region_reasoner_suite(self, tmp_path): csv_files = list(tmp_path.glob("**/jobs.csv")) assert len(csv_files) == 1 df = pd.read_csv(csv_files[0]) - # eval_suite should start with "region_reasoner" - assert all(s.startswith("region_reasoner") for s in df["eval_suite"]) - assert set(df["task_path"]) == {RR_TASK_NAME} + assert all(s.startswith("regiondial_bench") for s in df["eval_suite"]) + assert set(df["task_path"]) == {RD_TASK_REFCOCOG, RD_TASK_REFCOCOPLUS} # --------------------------------------------------------------------------- @@ -490,7 +558,7 @@ def _write_shard(self, shard_dir, idx, samples): path.write_text(json.dumps(samples)) def test_perfect_overlap(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard( tmp_path, @@ -505,7 +573,7 @@ def test_perfect_overlap(self, tmp_path): assert m["pass_rate_0.9"] == pytest.approx(1.0) def test_zero_overlap(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard( tmp_path, @@ -519,7 +587,7 @@ def test_zero_overlap(self, tmp_path): assert m["pass_rate_0.3"] == pytest.approx(0.0) def test_pass_rates_differ_across_thresholds(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard( tmp_path, @@ -530,14 +598,13 @@ def test_pass_rates_differ_across_thresholds(self, tmp_path): ], ) m = _aggregate_shards(str(tmp_path)) - # mask IoU=1.0 passes all; mask IoU=0.05 passes none assert m["pass_rate_0.3"] == pytest.approx(0.5) assert m["pass_rate_0.5"] == pytest.approx(0.5) assert m["pass_rate_0.7"] == pytest.approx(0.5) assert m["pass_rate_0.9"] == pytest.approx(0.5) def test_pass_rates_actually_differ(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard( tmp_path, @@ -548,14 +615,13 @@ def test_pass_rates_actually_differ(self, tmp_path): ], ) m = _aggregate_shards(str(tmp_path)) - # mask IoU=1.0 passes all; mask IoU=0.64 passes 0.3 and 0.5 but not 0.7, 0.9 assert m["pass_rate_0.3"] == pytest.approx(1.0) assert m["pass_rate_0.5"] == pytest.approx(1.0) assert m["pass_rate_0.7"] == pytest.approx(0.5) assert m["pass_rate_0.9"] == pytest.approx(0.5) def test_multiple_shards_aggregated(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard( tmp_path, @@ -572,18 +638,80 @@ def test_multiple_shards_aggregated(self, tmp_path): assert m["cIoU"] == pytest.approx(0.5) def test_no_shard_files_raises(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards with pytest.raises(RuntimeError, match="No shard output files"): _aggregate_shards(str(tmp_path)) def test_empty_shard_raises(self, tmp_path): - from oellm.contrib.region_reasoner.suite import _aggregate_shards + from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard(tmp_path, 0, []) with pytest.raises(RuntimeError, match="No samples found"): _aggregate_shards(str(tmp_path)) + def test_per_round_metrics_present(self, tmp_path): + """Samples with 'round' field produce per-round gIoU and bbox_AP keys.""" + from oellm.contrib.regiondial_bench.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [ + {"intersection": 100, "union": 100, "bbox_iou": 1.0, "round": 1}, + {"intersection": 50, "union": 100, "bbox_iou": 0.6, "round": 1}, + {"intersection": 0, "union": 100, "bbox_iou": 0.0, "round": 2}, + {"intersection": 80, "union": 100, "bbox_iou": 0.8, "round": 2}, + ], + ) + m = _aggregate_shards(str(tmp_path)) + # Per-round keys must exist + assert "gIoU_R1" in m + assert "gIoU_R2" in m + assert "bbox_AP_R1" in m + assert "bbox_AP_R2" in m + # R1: gIoU = mean(1.0, 0.5) = 0.75 + assert m["gIoU_R1"] == pytest.approx(0.75) + # R2: gIoU = mean(0.0, 0.8) = 0.4 + assert m["gIoU_R2"] == pytest.approx(0.4) + # R1 bbox_AP: both > 0.5 → 1.0 + assert m["bbox_AP_R1"] == pytest.approx(1.0) + # R2 bbox_AP: one >0.5 (0.8), one =0.0 → 0.5 + assert m["bbox_AP_R2"] == pytest.approx(0.5) + + def test_per_round_metrics_absent_without_round_field(self, tmp_path): + """Samples without 'round' field produce no per-round keys.""" + from oellm.contrib.regiondial_bench.suite import _aggregate_shards + + self._write_shard( + tmp_path, + 0, + [{"intersection": 100, "union": 100, "bbox_iou": 1.0}], + ) + m = _aggregate_shards(str(tmp_path)) + round_keys = [k for k in m if "_R" in k] + assert round_keys == [] + + def test_per_round_metrics_seven_rounds(self, tmp_path): + """All 7 rounds produce per-round metrics when present.""" + from oellm.contrib.regiondial_bench.suite import _aggregate_shards + + samples = [] + for rnd in range(1, 8): + samples.append( + { + "intersection": 100 - rnd * 10, + "union": 100, + "bbox_iou": (100 - rnd * 10) / 100, + "round": rnd, + } + ) + self._write_shard(tmp_path, 0, samples) + m = _aggregate_shards(str(tmp_path)) + for rnd in range(1, 8): + assert f"gIoU_R{rnd}" in m + assert f"bbox_AP_R{rnd}" in m + # --------------------------------------------------------------------------- # collect_results compatibility @@ -591,9 +719,9 @@ def test_empty_shard_raises(self, tmp_path): class TestCollectResultsCompatibility: - """Verify collect_results() parses RegionReasoner output without modification.""" + """Verify collect_results() parses RegionDial-Bench output without modification.""" - def test_collect_results_parses_region_reasoner_json(self, tmp_path): + def test_collect_results_parses_refcocog_json(self, tmp_path): import pandas as pd from oellm.main import collect_results @@ -601,11 +729,10 @@ def test_collect_results_parses_region_reasoner_json(self, tmp_path): results_dir = tmp_path / "results" results_dir.mkdir() - # Write a mock RegionReasoner output JSON (lmms-eval-compatible format) mock_output = { "model_name_or_path": "/cluster/models/RegionReasoner-7B", "results": { - RR_TASK_NAME: { + RD_TASK_REFCOCOG: { "gIoU": 0.42, "cIoU": 0.45, "bbox_AP": 0.38, @@ -613,7 +740,7 @@ def test_collect_results_parses_region_reasoner_json(self, tmp_path): "pass_rate_0.5": 0.55, } }, - "configs": {RR_TASK_NAME: {"num_fewshot": 0}}, + "configs": {RD_TASK_REFCOCOG: {"num_fewshot": 0}}, } (results_dir / "abc123.json").write_text(json.dumps(mock_output)) @@ -624,7 +751,38 @@ def test_collect_results_parses_region_reasoner_json(self, tmp_path): df = pd.read_csv(output_csv) assert len(df) == 1 row = df.iloc[0] - assert row["task"] == RR_TASK_NAME - assert row["metric_name"] in ("gIoU", "gIoU,none") # primary metric + assert row["task"] == RD_TASK_REFCOCOG + assert row["metric_name"] in ("gIoU", "gIoU,none") assert float(row["performance"]) == pytest.approx(0.42) assert row["model_name"] == "/cluster/models/RegionReasoner-7B" + + def test_collect_results_parses_refcocoplus_json(self, tmp_path): + import pandas as pd + + from oellm.main import collect_results + + results_dir = tmp_path / "results" + results_dir.mkdir() + + mock_output = { + "model_name_or_path": "/cluster/models/RegionReasoner-7B", + "results": { + RD_TASK_REFCOCOPLUS: { + "gIoU": 0.55, + "cIoU": 0.50, + "bbox_AP": 0.48, + } + }, + "configs": {RD_TASK_REFCOCOPLUS: {"num_fewshot": 0}}, + } + (results_dir / "def456.json").write_text(json.dumps(mock_output)) + + output_csv = str(tmp_path / "results.csv") + collect_results(str(tmp_path), output_csv=output_csv) + + assert Path(output_csv).exists() + df = pd.read_csv(output_csv) + assert len(df) == 1 + row = df.iloc[0] + assert row["task"] == RD_TASK_REFCOCOPLUS + assert float(row["performance"]) == pytest.approx(0.55) diff --git a/tests/test_registry.py b/tests/test_registry.py index 283a220d..b9ee0f14 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -6,17 +6,17 @@ class TestRegistryDiscovery: - def test_region_reasoner_is_discovered(self): - """The RegionReasoner suite must be auto-discovered from contrib/.""" + def test_regiondial_bench_is_discovered(self): + """The RegionDial-Bench suite must be auto-discovered from contrib/.""" suites = registry.get_all_suites() suite_names = [getattr(mod, "SUITE_NAME", None) for mod in suites] - assert "region_reasoner" in suite_names + assert "regiondial_bench" in suite_names def test_get_suite_returns_module(self): - mod = registry.get_suite("region_reasoner") + mod = registry.get_suite("regiondial_bench") assert mod is not None assert hasattr(mod, "SUITE_NAME") - assert mod.SUITE_NAME == "region_reasoner" + assert mod.SUITE_NAME == "regiondial_bench" def test_get_suite_unknown_raises_keyerror(self): with pytest.raises(KeyError, match="nonexistent_suite_xyz"): @@ -25,8 +25,7 @@ def test_get_suite_unknown_raises_keyerror(self): def test_keyerror_message_lists_known_suites(self): with pytest.raises(KeyError) as exc_info: registry.get_suite("nonexistent_suite_xyz") - # The error message should mention known suites to help diagnose typos - assert "region_reasoner" in str(exc_info.value) + assert "regiondial_bench" in str(exc_info.value) def test_get_all_suites_returns_list(self): suites = registry.get_all_suites() @@ -34,7 +33,7 @@ def test_get_all_suites_returns_list(self): assert len(suites) >= 1 def test_suite_has_required_protocol_attributes(self): - mod = registry.get_suite("region_reasoner") + mod = registry.get_suite("regiondial_bench") assert hasattr(mod, "SUITE_NAME"), "suite.py must expose SUITE_NAME" assert hasattr(mod, "TASK_GROUPS"), "suite.py must expose TASK_GROUPS" assert callable(getattr(mod, "run", None)), "suite.py must expose run()" @@ -44,19 +43,24 @@ def test_suite_has_required_protocol_attributes(self): class TestRegistryTaskGroupMerge: - def test_task_metrics_contains_region_reasoner(self): + def test_task_metrics_contains_regiondial_refcocog(self): merged = registry.get_all_task_groups() - assert "regionreasoner_refcocog" in merged.get("task_metrics", {}) + assert "regiondial_refcocog" in merged.get("task_metrics", {}) - def test_task_groups_contains_region_reasoner(self): + def test_task_metrics_contains_regiondial_refcocoplus(self): merged = registry.get_all_task_groups() - assert "region-reasoner" in merged.get("task_groups", {}) + assert "regiondial_refcocoplus" in merged.get("task_metrics", {}) + + def test_task_groups_contains_regiondial_bench(self): + merged = registry.get_all_task_groups() + assert "regiondial-bench" in merged.get("task_groups", {}) def test_merged_task_group_has_correct_suite(self): merged = registry.get_all_task_groups() - tg = merged["task_groups"]["region-reasoner"] - assert tg["suite"] == "region_reasoner" + tg = merged["task_groups"]["regiondial-bench"] + assert tg["suite"] == "regiondial_bench" def test_merged_primary_metric(self): merged = registry.get_all_task_groups() - assert merged["task_metrics"]["regionreasoner_refcocog"] == "gIoU" + assert merged["task_metrics"]["regiondial_refcocog"] == "gIoU" + assert merged["task_metrics"]["regiondial_refcocoplus"] == "gIoU" From b47ebf1eea98b650d78bf328f728e50ca6208448 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 30 Mar 2026 10:13:58 +0200 Subject: [PATCH 12/44] [Base][Sync with upstream] Add missing belebele for norwegian Sync additional task with upstream: 1. Add missing task (belebele for norwegian) --- oellm/resources/task-groups.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index d22d7082..05e6760a 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -134,6 +134,8 @@ task_groups: subset: spa_Latn - task: belebele_swe_Latn subset: swe_Latn + - task: belebele_nob_Latn + subset: nob_Latn flores-200-eu-to-eng: description: "Flores 200 EU to English translation" suite: lighteval From c888c6c8dd9a930ffe379a7349d5b0537a303c72 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Tue, 31 Mar 2026 14:39:54 +0200 Subject: [PATCH 13/44] update leonardo storage info documentation --- docs/LEONARDO.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index aa181263..8e51195e 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -115,14 +115,34 @@ uv pip install -e ./elliot-cli ## 5. Set HuggingFace Cache Directory -Compute nodes have no internet access, so all models and datasets must be pre-downloaded. Set `HF_HOME` to point to your work storage: +Compute nodes have no internet access, so all models and datasets must be pre-downloaded. Set `HF_HOME` to point to a storage area large enough to hold your models and datasets. + +> **Warning:** `$HOME` on Leonardo has a quota of only **50 GB** — far too small for most LLMs and multimodal datasets. Do **not** use `$HOME` as your HF cache directory. + +Leonardo provides several larger storage areas ([full details](https://docs.hpc.cineca.it/hpc/hpc_data_storage.html)): + +| Area | Quota | Purge policy | Notes | +|------|-------|--------------|-------| +| `$WORK` | 1 TB | 6 months post-project | Persistent, parallel I/O, **recommended** | +| `$FAST` | 1 TB | 6 months post-project | Faster I/O than `$WORK`, no extension option | +| `$SCRATCH` | 20 TB | Files purged after **40 days** of inactivity | Temporary only | + +**Recommended:** use `$WORK` for persistent caches (models you reuse across runs): ```zsh -mkdir -p $HOME/hf_cache -export HF_HOME="$HOME/hf_cache" +mkdir -p $WORK/hf_cache +export HF_HOME="$WORK/hf_cache" +echo 'export HF_HOME="$WORK/hf_cache"' >> ~/.bashrc source ~/.bashrc ``` +If you need more space for a single campaign and don't need the cache long-term, `$SCRATCH` (up to 20 TB) is an option — but files are **automatically deleted after 40 days of inactivity** and must not be kept alive artificially with `touch`: + +```zsh +mkdir -p $SCRATCH/hf_cache +export HF_HOME="$SCRATCH/hf_cache" +``` + --- ## 6. Running Evaluations From a9aa9bfd771bd4084b5c3e514bb6eec8e83a7c72 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Thu, 2 Apr 2026 12:06:03 +0200 Subject: [PATCH 14/44] [Contrib][RR bench] Fix per-round metric computation in RegionDial-Bench aggregation (#10) * Fix per-round metric computation in RegionDial-Bench aggregation * Update tests for RegionDial-Bench * Fix minor typos --- oellm/contrib/regiondial_bench/adapter.py | 2 ++ oellm/contrib/regiondial_bench/metrics.py | 4 +-- oellm/contrib/regiondial_bench/suite.py | 40 +++++++++++------------ tests/test_regiondial_bench.py | 29 ++++++++-------- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/oellm/contrib/regiondial_bench/adapter.py b/oellm/contrib/regiondial_bench/adapter.py index a257c5b3..08423631 100644 --- a/oellm/contrib/regiondial_bench/adapter.py +++ b/oellm/contrib/regiondial_bench/adapter.py @@ -25,6 +25,8 @@ def to_contrib_flags(self) -> str | None: name = Path(self._path).name.lower() if "regionreasoner" in name or "region_reasoner" in name: return "vision_reasoner" + if "qwen2.5" in name: + return "qwen2.5" if "qwen2" in name: return "qwen2" if "qwen" in name: diff --git a/oellm/contrib/regiondial_bench/metrics.py b/oellm/contrib/regiondial_bench/metrics.py index 17234336..9ae0dde8 100644 --- a/oellm/contrib/regiondial_bench/metrics.py +++ b/oellm/contrib/regiondial_bench/metrics.py @@ -23,7 +23,7 @@ Metrics ------- - **GIoU**: mean of per-sample mask IoU (intersection / union). -- **CIoU**: cumulative IoU — sum of all intersections / sum of all unions. +- **CIoU**: sum of all intersections / sum of all unions. - **BboxAP**: fraction of samples where bbox IoU > 0.5. - **PassRate**: fraction of samples where mask IoU > *threshold*. """ @@ -78,7 +78,7 @@ def compute(self, predictions: list[str], references: list[str]) -> float: class CIoU(BaseMetric): - """Cumulative IoU (cIoU as reported in RegionDial-Bench). + """cIoU as reported in RegionDial-Bench. Formula: ``sum(all intersections) / sum(all unions)``. """ diff --git a/oellm/contrib/regiondial_bench/suite.py b/oellm/contrib/regiondial_bench/suite.py index 31742385..b96cb190 100644 --- a/oellm/contrib/regiondial_bench/suite.py +++ b/oellm/contrib/regiondial_bench/suite.py @@ -196,9 +196,10 @@ def run( "--num_parts", "1", "--batch_size", - "2", + "1", "--task_router_model_path", "Ricky06662/TaskRouter-1.5B", + "--binarize_bbox_iou", ] logger.info("Starting shard %d/%d: %s", idx + 1, num_gpus, " ".join(cmd)) proc = subprocess.Popen(cmd, env=shard_env, cwd=str(Path(test_json).parent)) @@ -356,27 +357,26 @@ def _aggregate_shards(shard_dir: str) -> dict[str, float]: metrics[m.name] = val logger.debug("%s = %.4f", m.name, val) + # Infer per-round membership by counting each image_id's occurrence order in + # the output (mirrors calculate_iou_with_bbox_by_turns.py). The inference + # script emits turns in sequential order per image, so the k-th time an + # image_id appears corresponds to turn k (1-indexed). rounds_map: dict[int, list[str]] = defaultdict(list) + image_turn_counter: dict[str, int] = {} for sample_dict, sample_str in zip(all_samples, samples, strict=True): - rnd = sample_dict.get("round") - if rnd is not None: - rounds_map[int(rnd)].append(sample_str) - - if rounds_map: - per_round_metrics = [GIoU(), BboxAP()] - for rnd in sorted(rounds_map): - rnd_samples = rounds_map[rnd] - rnd_refs = [""] * len(rnd_samples) - for m in per_round_metrics: - val = m.compute(rnd_samples, rnd_refs) - metrics[f"{m.name}_R{rnd}"] = val - logger.debug("%s_R%d = %.4f", m.name, rnd, val) - else: - logger.warning( - "No 'round' field found in samples — skipping per-round breakdown. " - "Per-round metrics (R1–R7) require the inference script to output " - "a 'round' field in each sample." - ) + image_id = str(sample_dict.get("image_id", "")) + image_turn_counter[image_id] = image_turn_counter.get(image_id, 0) + 1 + rnd = image_turn_counter[image_id] + rounds_map[rnd].append(sample_str) + + per_round_metrics = [GIoU(), BboxAP()] + for rnd in sorted(rounds_map): + rnd_samples = rounds_map[rnd] + rnd_refs = [""] * len(rnd_samples) + for m in per_round_metrics: + val = m.compute(rnd_samples, rnd_refs) + metrics[f"{m.name}_R{rnd}"] = val + logger.debug("%s_R%d = %.4f", m.name, rnd, val) return metrics diff --git a/tests/test_regiondial_bench.py b/tests/test_regiondial_bench.py index 425cd966..49a2afc0 100644 --- a/tests/test_regiondial_bench.py +++ b/tests/test_regiondial_bench.py @@ -395,7 +395,7 @@ def test_detect_model_flags_region_reasoner_model(self, suite): assert suite.detect_model_flags("lmsdss/RegionReasoner-7B") == "vision_reasoner" def test_detect_model_flags_qwen2_model(self, suite): - assert suite.detect_model_flags("Qwen/Qwen2.5-VL-7B-Instruct") == "qwen2" + assert suite.detect_model_flags("Qwen/Qwen2.5-VL-7B-Instruct") == "qwen2.5" def test_detect_model_flags_qwen1_model(self, suite): assert suite.detect_model_flags("Qwen/Qwen-VL-Chat") == "qwen" @@ -476,7 +476,7 @@ def test_contrib_flags_region_reasoner(self, adapter_cls): def test_contrib_flags_qwen2(self, adapter_cls): cls, _ = adapter_cls - assert cls("Qwen/Qwen2.5-VL-7B").to_contrib_flags() == "qwen2" + assert cls("Qwen/Qwen2.5-VL-7B").to_contrib_flags() == "qwen2.5" def test_contrib_flags_qwen(self, adapter_cls): cls, _ = adapter_cls @@ -490,7 +490,7 @@ def test_detect_model_flags_delegates_to_adapter(self): import oellm.contrib.regiondial_bench.suite as s assert s.detect_model_flags("lmsdss/RegionReasoner-7B") == "vision_reasoner" - assert s.detect_model_flags("Qwen/Qwen2.5-VL-7B") == "qwen2" + assert s.detect_model_flags("Qwen/Qwen2.5-VL-7B") == "qwen2.5" # --------------------------------------------------------------------------- @@ -651,17 +651,20 @@ def test_empty_shard_raises(self, tmp_path): _aggregate_shards(str(tmp_path)) def test_per_round_metrics_present(self, tmp_path): - """Samples with 'round' field produce per-round gIoU and bbox_AP keys.""" + """Two images with two turns each produce per-round gIoU and bbox_AP keys.""" from oellm.contrib.regiondial_bench.suite import _aggregate_shards + # Turns are consecutive per image (img1 T1, img1 T2, img2 T1, img2 T2). + # The turn counter assigns: first occurrence of each image_id → R1, + # second occurrence → R2. self._write_shard( tmp_path, 0, [ - {"intersection": 100, "union": 100, "bbox_iou": 1.0, "round": 1}, - {"intersection": 50, "union": 100, "bbox_iou": 0.6, "round": 1}, - {"intersection": 0, "union": 100, "bbox_iou": 0.0, "round": 2}, - {"intersection": 80, "union": 100, "bbox_iou": 0.8, "round": 2}, + {"image_id": "img1", "intersection": 100, "union": 100, "bbox_iou": 1.0}, + {"image_id": "img1", "intersection": 0, "union": 100, "bbox_iou": 0.0}, + {"image_id": "img2", "intersection": 50, "union": 100, "bbox_iou": 0.6}, + {"image_id": "img2", "intersection": 80, "union": 100, "bbox_iou": 0.8}, ], ) m = _aggregate_shards(str(tmp_path)) @@ -679,18 +682,18 @@ def test_per_round_metrics_present(self, tmp_path): # R2 bbox_AP: one >0.5 (0.8), one =0.0 → 0.5 assert m["bbox_AP_R2"] == pytest.approx(0.5) - def test_per_round_metrics_absent_without_round_field(self, tmp_path): - """Samples without 'round' field produce no per-round keys.""" + def test_per_round_metrics_always_present(self, tmp_path): + """Per-round keys are always produced — turns are inferred from image_id order.""" from oellm.contrib.regiondial_bench.suite import _aggregate_shards self._write_shard( tmp_path, 0, - [{"intersection": 100, "union": 100, "bbox_iou": 1.0}], + [{"image_id": "img1", "intersection": 100, "union": 100, "bbox_iou": 1.0}], ) m = _aggregate_shards(str(tmp_path)) - round_keys = [k for k in m if "_R" in k] - assert round_keys == [] + assert "gIoU_R1" in m + assert "bbox_AP_R1" in m def test_per_round_metrics_seven_rounds(self, tmp_path): """All 7 rounds produce per-round metrics when present.""" From 484bfb9d62a0b94fb9a8b6bdb642cc38035617bd Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Wed, 8 Apr 2026 00:33:35 +0200 Subject: [PATCH 15/44] fixes --- oellm/resources/task-groups.yaml | 6 +++--- oellm/utils.py | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 947d388a..65a53913 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -240,7 +240,7 @@ task_groups: description: "EU Language GSM benchmarks in Aya Expanse" suite: lm-eval-harness n_shots: [5] - dataset: jbross-ibm-research/mgsm + dataset: juletxara/mgsm tasks: - task: mgsm_native_cot_en subset: en @@ -329,7 +329,7 @@ task_groups: - task: vqav2_val_all dataset: HuggingFaceM4/VQAv2 - task: mmbench_en_dev - dataset: HuggingFaceM4/MMBench_00 + dataset: lmms-lab/MMBench - task: mmmu_val dataset: MMMU/MMMU - task: chartqa @@ -358,7 +358,7 @@ task_groups: n_shots: [0] tasks: - task: mmbench_en_dev - dataset: HuggingFaceM4/MMBench_00 + dataset: lmms-lab/MMBench image-mmmu: description: "MMMU massive multi-discipline multimodal understanding via lmms-eval" diff --git a/oellm/utils.py b/oellm/utils.py index 5bd74dbf..27bde7d5 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -309,12 +309,29 @@ def _process_model_paths(models: Iterable[str]): ) logging.debug(e) else: + cache_dir = ( + Path(os.getenv("HF_HOME")) / "hub" + if "HF_HOME" in os.environ + else None + ) + try: + from huggingface_hub import try_to_load_from_cache + + cached = try_to_load_from_cache( + model, "config.json", cache_dir=cache_dir + ) + if isinstance(cached, str): + logging.info( + f"Model '{model}' already cached, skipping download" + ) + per_model_paths.append(model) + continue + except Exception: + pass status.update(f"Downloading '{model}' ({idx}/{len(models_list)})") snapshot_download( repo_id=model, - cache_dir=Path(os.getenv("HF_HOME")) / "hub" - if "HF_HOME" in os.environ - else None, + cache_dir=cache_dir, ) per_model_paths.append(model) From ff79828fee95de59a889b65e5addd671b536fed3 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Wed, 8 Apr 2026 00:45:43 +0200 Subject: [PATCH 16/44] fix tests --- tests/test_image_task_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py index eb8b50cb..1e90b57b 100644 --- a/tests/test_image_task_groups.py +++ b/tests/test_image_task_groups.py @@ -28,7 +28,7 @@ EXPECTED_DATASETS = { "HuggingFaceM4/VQAv2", - "HuggingFaceM4/MMBench_00", + "lmms-lab/MMBench", "MMMU/MMMU", "HuggingFaceM4/ChartQA", "eliolio/docvqa", From d6a417357ec3884f1094ce0f818a0835ee043398 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Thu, 9 Apr 2026 14:30:37 +0200 Subject: [PATCH 17/44] [Base][Refactor] Switch CLI to Typer, added runs via config Switch CLI to Typer, added runs via config --- README.md | 44 +- docs/CONTAINERS.md | 2 +- docs/LEONARDO.md | 4 +- docs/TASKS.md | 4 +- docs/VENV.md | 22 +- eval.yaml | 17 + oellm/config.py | 401 ++++++++++++++ oellm/contrib/CONTRIBUTING.md | 4 +- oellm/contrib/README.md | 4 +- oellm/contrib/regiondial_bench/README.md | 22 +- oellm/main.py | 640 +++++++++-------------- oellm/resources/task-groups.yaml | 4 +- oellm/results.py | 85 ++- oellm/runner.py | 110 ++++ oellm/scheduler.py | 440 ++++++++++++++++ pyproject.toml | 2 +- tests/test_collect_results.py | 54 ++ tests/test_compare.py | 206 ++++++++ tests/test_eval_command.py | 217 ++++++++ tests/test_eval_config.py | 400 ++++++++++++++ tests/test_image_task_groups.py | 14 +- tests/test_list_tasks.py | 55 ++ tests/test_regiondial_bench.py | 8 +- tests/test_reporter.py | 117 +++++ tests/test_runner.py | 185 +++++++ tests/test_schedule_evals.py | 14 +- 26 files changed, 2614 insertions(+), 461 deletions(-) create mode 100644 eval.yaml create mode 100644 oellm/config.py create mode 100644 oellm/runner.py create mode 100644 oellm/scheduler.py create mode 100644 tests/test_compare.py create mode 100644 tests/test_eval_command.py create mode 100644 tests/test_eval_config.py create mode 100644 tests/test_list_tasks.py create mode 100644 tests/test_reporter.py create mode 100644 tests/test_runner.py diff --git a/README.md b/README.md index b014274e..799a0af6 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,13 @@ uv tool install -p 3.12 git+https://github.com/elliot-project/elliot-cli.git # Run evaluations using a task group oellm schedule-eval \ --models "EleutherAI/pythia-160m" \ - --task_groups "open-sci-0.01" + --task-groups "open-sci-0.01" # Image evaluation (requires venv with lmms-eval) oellm schedule-eval \ --models "llava-hf/llava-1.5-7b-hf" \ - --task_groups "image-vqa" \ - --venv_path ~/elliot-venv + --task-groups "image-vqa" \ + --venv-path ~/elliot-venv ``` This will automatically: @@ -40,7 +40,7 @@ This will automatically: - Pre-download datasets for known tasks (see warning below) - Generate and submit a SLURM job array with appropriate cluster-specific resources and using containers built for this cluster -For custom environments instead of containers, pass `--venv_path` (see [docs/VENV.md](docs/VENV.md)). +For custom environments instead of containers, pass `--venv-path` (see [docs/VENV.md](docs/VENV.md)). ## Task Groups @@ -85,18 +85,18 @@ Community-contributed benchmarks that run outside the standard evaluation engine # Run all 8 image benchmarks oellm schedule-eval \ --models "llava-hf/llava-1.5-7b-hf" \ - --task_groups "image-vqa" \ - --venv_path ~/elliot-venv + --task-groups "image-vqa" \ + --venv-path ~/elliot-venv # Mix image and text benchmarks in one submission oellm schedule-eval \ --models "llava-hf/llava-1.5-7b-hf" \ - --task_groups "image-mmbench,open-sci-0.01" \ - --venv_path ~/elliot-venv + --task-groups "image-mmbench,open-sci-0.01" \ + --venv-path ~/elliot-venv # Use multiple task groups or a super group -oellm schedule-eval --models "model-name" --task_groups "belebele-eu-5-shot,global-mmlu-eu" -oellm schedule-eval --models "model-name" --task_groups "oellm-multilingual" +oellm schedule-eval --models "model-name" --task-groups "belebele-eu-5-shot,global-mmlu-eu" +oellm schedule-eval --models "model-name" --task-groups "oellm-multilingual" ``` ## Running Locally (without SLURM) @@ -111,8 +111,8 @@ uv pip install lm-eval torch transformers accelerate "datasets<4.0.0" oellm schedule-eval \ --models "EleutherAI/pythia-160m" \ --tasks "gsm8k" \ - --n_shot 0 \ - --venv_path .venv \ + --n-shot 0 \ + --venv-path .venv \ --local true \ --limit 1 ``` @@ -121,16 +121,16 @@ Results are written to `./oellm-output//results/`. ## SLURM Overrides -Override cluster defaults (partition, account, time limit, etc.) with `--slurm_template_var` (JSON object): +Override cluster defaults (partition, account, time limit, etc.) with `--slurm-template-var` (JSON object): ```bash # Use a different partition (e.g. dev-g on LUMI when small-g is crowded) -oellm schedule-eval --models "model-name" --task_groups "open-sci-0.01" \ - --slurm_template_var '{"PARTITION":"dev-g"}' +oellm schedule-eval --models "model-name" --task-groups "open-sci-0.01" \ + --slurm-template-var '{"PARTITION":"dev-g"}' # Multiple overrides: partition, account, time limit, GPUs -oellm schedule-eval --models "model-name" --task_groups "open-sci-0.01" \ - --slurm_template_var '{"PARTITION":"dev-g","ACCOUNT":"myproject","TIME":"02:00:00","GPUS_PER_NODE":2}' +oellm schedule-eval --models "model-name" --task-groups "open-sci-0.01" \ + --slurm-template-var '{"PARTITION":"dev-g","ACCOUNT":"myproject","TIME":"02:00:00","GPUS_PER_NODE":2}' ``` Use exact env var names: `PARTITION`, `ACCOUNT`, `GPUS_PER_NODE`. `TIME` (HH:MM:SS) overrides the time limit. @@ -141,18 +141,18 @@ Use exact env var names: `PARTITION`, `ACCOUNT`, `GPUS_PER_NODE`. `TIME` (HH:MM: If you use custom tasks via `--tasks` that are not in the task groups registry, the CLI will attempt to look them up but **cannot guarantee the datasets will be cached**. This may cause failures on compute nodes that don't have network access. -**Recommendation:** Use `--task_groups` when possible, or ensure your custom task datasets are already cached in `$HF_HOME` before scheduling. +**Recommendation:** Use `--task-groups` when possible, or ensure your custom task datasets are already cached in `$HF_HOME` before scheduling. ## Collecting Results ```bash # Basic collection -oellm collect-results --results_dir /path/to/eval-output-dir +oellm collect-results --results-dir /path/to/eval-output-dir # Check for missing evaluations and create a CSV for re-running them -oellm collect-results --results_dir /path/to/eval-output-dir --check true --output_csv results.csv +oellm collect-results --results-dir /path/to/eval-output-dir --check true --output-csv results.csv # Re-schedule failed jobs -oellm schedule-eval --eval_csv_path results_missing.csv +oellm schedule-eval --eval-csv-path results_missing.csv ``` ## Installation @@ -197,7 +197,7 @@ uv sync --extra dev uv run pytest tests/ -v # Download-only mode for testing -uv run oellm schedule-eval --models "EleutherAI/pythia-160m" --task_groups "open-sci-0.01" --download_only +uv run oellm schedule-eval --models "EleutherAI/pythia-160m" --task-groups "open-sci-0.01" --download-only ``` ## Documentation diff --git a/docs/CONTAINERS.md b/docs/CONTAINERS.md index 1fe61be3..31d6e7c9 100644 --- a/docs/CONTAINERS.md +++ b/docs/CONTAINERS.md @@ -18,7 +18,7 @@ Images are compressed with zstd (level 3) via mksquashfs for a good balance of s Image benchmarks (`suite: lmms_eval`) require `lmms-eval` to be available in the execution environment. There are two ways to provide it: **Option 1 — Custom venv (recommended for development):** -Install `lmms-eval` via `requirements-venv.txt` and pass `--venv_path` to the CLI. See [VENV.md](VENV.md) for setup instructions. +Install `lmms-eval` via `requirements-venv.txt` and pass `--venv-path` to the CLI. See [VENV.md](VENV.md) for setup instructions. **Option 2 — Container with lmms-eval:** Build a container `.def` file that includes `lmms-eval` alongside `lm-eval`: diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index 8e51195e..1f51de46 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -151,13 +151,13 @@ export HF_HOME="$SCRATCH/hf_cache" # Run evaluations using a task group (recommended) oellm schedule-eval \ --models "microsoft/DialoGPT-medium,EleutherAI/pythia-160m" \ - --task_groups "open-sci-0.01" + --task-groups "open-sci-0.01" # Or specify individual tasks oellm schedule-eval \ --models "EleutherAI/pythia-160m" \ --tasks "hellaswag,mmlu" \ - --n_shot 5 + --n-shot 5 ``` --- diff --git a/docs/TASKS.md b/docs/TASKS.md index 86275028..d6ee1bc6 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -49,7 +49,7 @@ task_groups: 2. Use it: ```bash -oellm schedule-eval --models "model-name" --task_groups "my-benchmark" +oellm schedule-eval --models "model-name" --task-groups "my-benchmark" ``` ## Adding an Image Task Group @@ -72,7 +72,7 @@ task_groups: Run with: ```bash -oellm schedule-eval --models "path/to/vlm" --task_groups "my-image-benchmark" +oellm schedule-eval --models "path/to/vlm" --task-groups "my-image-benchmark" ``` The lmms-eval model adapter (e.g. `llava_hf`, `qwen2_vl`) is auto-detected diff --git a/docs/VENV.md b/docs/VENV.md index 8215e738..bce8ac43 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -2,7 +2,7 @@ ## Overview -Instead of using pre-built containers, you can run evaluations with your own Python virtual environment by passing `--venv_path`. +Instead of using pre-built containers, you can run evaluations with your own Python virtual environment by passing `--venv-path`. ## Setup @@ -32,14 +32,14 @@ Instead of using pre-built containers, you can run evaluations with your own Pyt # Text evaluation oellm schedule-eval \ --models HuggingFaceTB/SmolLM2-135M-Instruct \ - --task_groups open-sci-0.01 \ - --venv_path /path/to/.venv + --task-groups open-sci-0.01 \ + --venv-path /path/to/.venv # Image evaluation (lmms-eval) oellm schedule-eval \ --models path/to/vlm \ - --task_groups image-vqa \ - --venv_path /path/to/.venv + --task-groups image-vqa \ + --venv-path /path/to/.venv ``` ## Why Multiple Install Steps? @@ -68,9 +68,9 @@ uv pip install --python dclm-core-venv/bin/python -r requirements-venv-dclm.txt ```bash oellm schedule-eval \ --models Qwen/Qwen3-0.6B-Base \ - --task_groups dclm-core-22 \ - --venv_path dclm-core-venv \ - --skip_checks true + --task-groups dclm-core-22 \ + --venv-path dclm-core-venv \ + --skip-checks true ``` ## Evalchemy (reasoning) @@ -98,9 +98,9 @@ We use [Ali's fork](https://github.com/Ali-Elganzory/evalchemy) which includes a export HF_ALLOW_CODE_EVAL=1 # required by MBPP EVALCHEMY_DIR=$(pwd)/evalchemy oellm schedule-eval \ --models HuggingFaceTB/SmolLM2-135M \ - --task_groups reasoning \ - --venv_path evalchemy-venv \ - --skip_checks true + --task-groups reasoning \ + --venv-path evalchemy-venv \ + --skip-checks true ``` > **Note:** `HF_ALLOW_CODE_EVAL=1` is required because MBPP (run via lm-eval-harness) uses HuggingFace's `code_eval` metric which executes model-generated code. The evalchemy benchmarks (GPQADiamond, MATH500, LiveCodeBench) do not require this variable as they handle code execution safely through internal guards. diff --git a/eval.yaml b/eval.yaml new file mode 100644 index 00000000..122bbe0c --- /dev/null +++ b/eval.yaml @@ -0,0 +1,17 @@ +# ELLIOT evaluation config — copy and edit this file for your run. + +models: + - "EleutherAI/pythia-70m" + +task_groups: + - "open-sci-0.01" + # - "image-vqa" # P0 image benchmarks (VQAv2, MMBench, MMMU, …) + # - "dclm-core-22" + # - "belebele-eu-5-shot" + +slurm: + time_limit: "12:00:00" + max_array_len: 128 + # partition: "boost_usr_prod" + # account: "OELLM_prod2026" + # gpus_per_node: 4 diff --git a/oellm/config.py b/oellm/config.py new file mode 100644 index 00000000..70e6bf35 --- /dev/null +++ b/oellm/config.py @@ -0,0 +1,401 @@ +"""Typed evaluation configuration with YAML loading and CLI override support. + +Users can define a reusable, version-controllable YAML config file instead of +passing a dozen CLI flags every time. Every field is overridable from the CLI +(CLI wins). When no ``--config`` is given the existing CLI-only workflow is +unchanged — ``schedule_evals`` builds an ``EvalConfig`` internally from its +loose parameters. +""" + +from __future__ import annotations + +import logging +from dataclasses import MISSING, dataclass, field, fields +from pathlib import Path +from typing import Any + +import yaml + + +@dataclass +class SlurmOverrides: + """SLURM-specific overrides that map to ``slurm_template_var`` JSON.""" + + partition: str | None = None + account: str | None = None + gpus_per_node: int | None = None + time_limit: str | None = None + max_array_len: int = 128 + + def to_template_var_dict(self) -> dict[str, str]: + """Return a dict suitable for ``slurm_template_var`` JSON consumption.""" + d: dict[str, str] = {} + if self.partition is not None: + d["PARTITION"] = self.partition + if self.account is not None: + d["ACCOUNT"] = self.account + if self.gpus_per_node is not None: + d["GPUS_PER_NODE"] = str(self.gpus_per_node) + if self.time_limit is not None: + d["TIME"] = self.time_limit + return d + + +@dataclass +class ModelConfig: + """Named model entry for the ``models:`` YAML key. + + Plain strings and dict entries (``path`` + optional ``name``) are both + accepted; see :meth:`EvalConfig.from_yaml` for examples. + """ + + path: str + name: str | None = None + + +@dataclass +class EvalConfig: + """Unified evaluation configuration. + + Can be constructed from: + * a YAML file via :meth:`from_yaml` + * raw CLI keyword arguments via :meth:`from_cli_kwargs` + * merging both (CLI wins) via :meth:`merge` + """ + + # ---- what to evaluate ---- + models: list[str | ModelConfig] | None = None + tasks: list[str] | None = None + task_groups: list[str] | None = None + n_shot: list[int] | None = None + eval_csv_path: str | None = None + + # ---- execution flags ---- + limit: int | None = None + verbose: bool = False + download_only: bool = False + dry_run: bool = False + skip_checks: bool = False + trust_remote_code: bool = True + venv_path: str | None = None + lm_eval_include_path: str | None = None + local: bool = False + + # ---- SLURM overrides ---- + slurm: SlurmOverrides = field(default_factory=SlurmOverrides) + + # ------------------------------------------------------------------ + # Construction helpers + # ------------------------------------------------------------------ + + @classmethod + def from_yaml(cls, path: str | Path) -> EvalConfig: + """Load config from a YAML file. + + Example YAML:: + + models: + - "llava-hf/llava-1.5-7b-hf" + - "Qwen/Qwen2-VL-7B" + task_groups: + - "image-vqa" + n_shot: 0 + trust_remote_code: true + venv_path: "~/elliot-venv" + slurm: + max_array_len: 64 + partition: "gpu" + time_limit: "06:00:00" + """ + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"Config file not found: {path}") + + with open(path) as f: + raw: dict[str, Any] = yaml.safe_load(f) or {} + + return cls._from_dict(raw) + + @classmethod + def from_cli_kwargs( + cls, + *, + models: str | None = None, + tasks: str | None = None, + task_groups: str | None = None, + n_shot: int | list[int] | None = None, + eval_csv_path: str | None = None, + max_array_len: int = 128, + limit: int | None = None, + verbose: bool = False, + download_only: bool = False, + dry_run: bool = False, + skip_checks: bool = False, + trust_remote_code: bool = True, + venv_path: str | None = None, + lm_eval_include_path: str | None = None, + local: bool = False, + slurm_template_var: str | None = None, + ) -> EvalConfig: + """Build an ``EvalConfig`` from the loose CLI parameters. + + This is the bridge that keeps the existing CLI signature 100 % + backward-compatible. + """ + import json + + models_list = ( + [m.strip() for m in models.split(",") if m.strip()] + if isinstance(models, str) + else None + ) + tasks_list = ( + [t.strip() for t in tasks.split(",") if t.strip()] + if isinstance(tasks, str) + else None + ) + groups_list = ( + [g.strip() for g in task_groups.split(",") if g.strip()] + if isinstance(task_groups, str) + else None + ) + n_shot_list: list[int] | None = None + if isinstance(n_shot, int): + n_shot_list = [n_shot] + elif isinstance(n_shot, list): + n_shot_list = n_shot + + slurm = SlurmOverrides(max_array_len=max_array_len) + if slurm_template_var: + try: + opts = json.loads(slurm_template_var) + except json.JSONDecodeError as e: + raise ValueError( + f"slurm_template_var must be a valid JSON object: {e}" + ) from e + if not isinstance(opts, dict): + raise ValueError( + "slurm_template_var must be a JSON object, e.g. " + '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00"}' + ) + slurm.partition = opts.get("PARTITION", opts.get("partition")) + slurm.account = opts.get("ACCOUNT", opts.get("account")) + gpus = opts.get("GPUS_PER_NODE", opts.get("gpus_per_node")) + if gpus is not None: + slurm.gpus_per_node = int(gpus) + slurm.time_limit = opts.get("TIME", opts.get("time_limit")) + + return cls( + models=models_list, + tasks=tasks_list, + task_groups=groups_list, + n_shot=n_shot_list, + eval_csv_path=eval_csv_path, + limit=limit, + verbose=verbose, + download_only=download_only, + dry_run=dry_run, + skip_checks=skip_checks, + trust_remote_code=trust_remote_code, + venv_path=venv_path, + lm_eval_include_path=lm_eval_include_path, + local=local, + slurm=slurm, + ) + + @classmethod + def _from_dict(cls, raw: dict[str, Any]) -> EvalConfig: + """Construct from a raw dict (YAML or programmatic).""" + slurm_raw = raw.get("slurm", {}) or {} + slurm = SlurmOverrides( + partition=slurm_raw.get("partition"), + account=slurm_raw.get("account"), + gpus_per_node=_optional_int(slurm_raw.get("gpus_per_node")), + time_limit=slurm_raw.get("time_limit"), + max_array_len=int(slurm_raw.get("max_array_len", 128)), + ) + + # Normalise scalar → list for models / tasks / task_groups / n_shot + models = _ensure_model_list(raw.get("models")) + tasks = _ensure_str_list(raw.get("tasks")) + task_groups = _ensure_str_list(raw.get("task_groups")) + n_shot = _ensure_int_list(raw.get("n_shot")) + + return cls( + models=models, + tasks=tasks, + task_groups=task_groups, + n_shot=n_shot, + eval_csv_path=raw.get("eval_csv_path"), + limit=_optional_int(raw.get("limit")), + verbose=bool(raw.get("verbose", False)), + download_only=bool(raw.get("download_only", False)), + dry_run=bool(raw.get("dry_run", False)), + skip_checks=bool(raw.get("skip_checks", False)), + trust_remote_code=bool(raw.get("trust_remote_code", True)), + venv_path=raw.get("venv_path"), + lm_eval_include_path=raw.get("lm_eval_include_path"), + local=bool(raw.get("local", False)), + slurm=slurm, + ) + + def merge(self, cli: EvalConfig) -> EvalConfig: + """Return a new config where *cli* values override *self* (the YAML base). + + A CLI field is considered "set" when it differs from the class default. + """ + merged_kwargs: dict[str, Any] = {} + for f in fields(self): + yaml_val = getattr(self, f.name) + cli_val = getattr(cli, f.name) + default_val = _field_default(f) + + if f.name == "slurm": + merged_kwargs["slurm"] = _merge_slurm(yaml_val, cli_val) + elif cli_val != default_val: + # CLI explicitly set — use it + merged_kwargs[f.name] = cli_val + else: + merged_kwargs[f.name] = yaml_val + + return EvalConfig(**merged_kwargs) + + # ------------------------------------------------------------------ + # Validation + # ------------------------------------------------------------------ + + def validate(self) -> None: + """Raise ``ValueError`` on invalid or contradictory configuration.""" + if self.eval_csv_path: + if self.models or self.tasks or self.task_groups or self.n_shot: + raise ValueError( + "Cannot specify models, tasks, task_groups, or n_shot " + "when eval_csv_path is provided." + ) + if not Path(self.eval_csv_path).exists(): + raise FileNotFoundError( + f"eval_csv_path does not exist: {self.eval_csv_path}" + ) + return # CSV mode — nothing else to validate + + if not self.models: + raise ValueError("At least one model must be specified.") + + if self.task_groups is None and self.tasks is None: + raise ValueError( + "Either task_groups or tasks must be specified (or use eval_csv_path)." + ) + + if self.tasks and not self.n_shot: + raise ValueError("n_shot is required when specifying individual tasks.") + + if self.n_shot: + for s in self.n_shot: + if not isinstance(s, int) or s < 0: + raise ValueError( + f"n_shot values must be non-negative integers, got: {s}" + ) + + if self.venv_path: + venv = Path(self.venv_path).expanduser() + if not (venv / "bin" / "python").exists(): + logging.warning( + f"venv_path '{self.venv_path}' does not contain bin/python " + f"— this may fail on the cluster." + ) + + # ------------------------------------------------------------------ + # Convenience + # ------------------------------------------------------------------ + + def _model_paths(self) -> list[str]: + """Return plain path strings, unwrapping any ModelConfig objects.""" + if not self.models: + return [] + return [m.path if isinstance(m, ModelConfig) else m for m in self.models] + + @property + def slurm_template_var_json(self) -> str | None: + """Return the JSON string for ``slurm_template_var``, or None if empty.""" + import json + + d = self.slurm.to_template_var_dict() + return json.dumps(d) if d else None + + +# ------------------------------------------------------------------ +# Internal helpers +# ------------------------------------------------------------------ + + +def _ensure_model_list(val: Any) -> list[str | ModelConfig] | None: + """Normalise the ``models:`` YAML value to a list of str or ModelConfig.""" + if val is None: + return None + items = val if isinstance(val, list) else [val] + result: list[str | ModelConfig] = [] + for item in items: + if isinstance(item, dict): + result.append(ModelConfig(path=item["path"], name=item.get("name"))) + else: + s = str(item).strip() + if s: + result.append(s) + return result or None + + +def _ensure_str_list(val: Any) -> list[str] | None: + if val is None: + return None + if isinstance(val, str): + return [s.strip() for s in val.split(",") if s.strip()] + if isinstance(val, list): + return [str(v).strip() for v in val if str(v).strip()] + return [str(val)] + + +def _ensure_int_list(val: Any) -> list[int] | None: + if val is None: + return None + if isinstance(val, int): + return [val] + if isinstance(val, list): + return [int(v) for v in val] + return [int(val)] + + +def _optional_int(val: Any) -> int | None: + if val is None: + return None + return int(val) + + +def _field_default(f: Any) -> Any: + """Return the default value for a dataclass field.""" + if f.default is not MISSING: + return f.default + if f.default_factory is not MISSING: + return f.default_factory() + return None + + +def _merge_slurm(yaml_slurm: SlurmOverrides, cli_slurm: SlurmOverrides) -> SlurmOverrides: + """Merge two SlurmOverrides — CLI wins when non-default.""" + default = SlurmOverrides() + return SlurmOverrides( + partition=cli_slurm.partition + if cli_slurm.partition != default.partition + else yaml_slurm.partition, + account=cli_slurm.account + if cli_slurm.account != default.account + else yaml_slurm.account, + gpus_per_node=cli_slurm.gpus_per_node + if cli_slurm.gpus_per_node != default.gpus_per_node + else yaml_slurm.gpus_per_node, + time_limit=cli_slurm.time_limit + if cli_slurm.time_limit != default.time_limit + else yaml_slurm.time_limit, + max_array_len=cli_slurm.max_array_len + if cli_slurm.max_array_len != default.max_array_len + else yaml_slurm.max_array_len, + ) diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index 46927812..f1a5992d 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -25,8 +25,8 @@ task_groups: ```bash oellm schedule-eval \ --models org/MyModel \ - --task_groups my-benchmark \ - --venv_path ~/elliot-venv + --task-groups my-benchmark \ + --venv-path ~/elliot-venv ``` Supported `suite` values: diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md index 21250a6b..bd08efa0 100644 --- a/oellm/contrib/README.md +++ b/oellm/contrib/README.md @@ -17,8 +17,8 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). ```bash oellm schedule-eval \ --models lmsdss/RegionReasoner-7B \ - --task_groups regiondial-bench \ - --venv_path ~/elliot-venv + --task-groups regiondial-bench \ + --venv-path ~/elliot-venv ``` Requires cluster-specific setup (`REGION_REASONER_DIR`, etc.). See the full [RegionDial-Bench README](regiondial_bench/README.md) for prerequisites and configuration. diff --git a/oellm/contrib/regiondial_bench/README.md b/oellm/contrib/regiondial_bench/README.md index c25c2e72..b540d6ef 100644 --- a/oellm/contrib/regiondial_bench/README.md +++ b/oellm/contrib/regiondial_bench/README.md @@ -100,22 +100,22 @@ Three task groups are available: # Both splits oellm schedule-eval \ --models lmsdss/RegionReasoner-7B \ - --task_groups regiondial-bench \ - --venv_path ~/elliot-venv + --task-groups regiondial-bench \ + --venv-path ~/elliot-venv # Single split oellm schedule-eval \ --models lmsdss/RegionReasoner-7B \ - --task_groups regiondial-refcocog \ - --venv_path ~/elliot-venv + --task-groups regiondial-refcocog \ + --venv-path ~/elliot-venv ``` ### Collecting results ```bash oellm collect-results \ - --eval_output_dir /path/to/evals \ - --output results.csv + --eval-output-dir /path/to/evals \ + --output-csv results.csv ``` The primary metric in the CSV is **gIoU**. Per-round metrics (e.g. @@ -133,8 +133,8 @@ model, just pass it to `--models`: ```bash oellm schedule-eval \ --models Qwen/Qwen2.5-VL-7B-Instruct \ - --task_groups regiondial-bench \ - --venv_path ~/elliot-venv + --task-groups regiondial-bench \ + --venv-path ~/elliot-venv ``` The model type is resolved as follows: @@ -150,9 +150,9 @@ To evaluate multiple models in one go: ```bash oellm schedule-eval \ - --models lmsdss/RegionReasoner-7B Qwen/Qwen2.5-VL-7B-Instruct \ - --task_groups regiondial-bench \ - --venv_path ~/elliot-venv + --models "lmsdss/RegionReasoner-7B,Qwen/Qwen2.5-VL-7B-Instruct" \ + --task-groups regiondial-bench \ + --venv-path ~/elliot-venv ``` > If your model name does not match any pattern above and requires a specific diff --git a/oellm/main.py b/oellm/main.py index 19a32244..d19eb977 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -1,52 +1,41 @@ -import json import logging -import math -import os -import re -import subprocess -from datetime import datetime -from importlib.resources import files from pathlib import Path -from string import Template -import pandas as pd -from jsonargparse import auto_cli +import typer +from typer import rich_utils -from oellm.constants import EvaluationJob, detect_lmms_model_type +from oellm.config import EvalConfig from oellm.results import collect_results -from oellm.task_groups import ( - _collect_dataset_specs, - _collect_hf_dataset_files, - _collect_hf_model_repos, - _expand_task_groups, - _lookup_dataset_specs_for_tasks, +from oellm.utils import _filter_warnings, _setup_logging + +# Override Typer's default cyan colour scheme with colours that are readable +# on both light (white) and dark terminal backgrounds. +rich_utils.COLOR_OPTIONS_PANEL_TITLE = "bold blue" +rich_utils.COLOR_ARGUMENTS_PANEL_TITLE = "bold blue" +rich_utils.COLOR_COMMANDS_PANEL_TITLE = "bold blue" +rich_utils.STYLE_OPTION = "bold blue" +rich_utils.STYLE_SWITCH = "bold dark_green" +rich_utils.STYLE_NEGATIVE_OPTION = "bold magenta" +rich_utils.STYLE_NEGATIVE_SWITCH = "bold magenta" +rich_utils.STYLE_METAVAR = "dark_orange3" +rich_utils.STYLE_OPTION_DEFAULT = "dim" + +app = typer.Typer( + name="oellm", + help="ELLIOT: Multi-cluster evaluation tool for language models", + no_args_is_help=True, + pretty_exceptions_show_locals=False, ) -from oellm.utils import ( - _ensure_runtime_environment, - _expand_local_model_paths, - _filter_warnings, - _load_cluster_env, - _num_jobs_in_queue, - _pre_download_datasets_from_specs, - _pre_download_hf_dataset_files, - _pre_download_hf_model_repos, - _process_model_paths, - _setup_logging, - capture_third_party_output_from_kwarg, -) - -# Backward-compatible alias -_detect_lmms_model_type = detect_lmms_model_type -@capture_third_party_output_from_kwarg("verbose") def schedule_evals( models: str | None = None, tasks: str | None = None, task_groups: str | None = None, - n_shot: int | list[int] | None = None, + n_shot: list[int] | None = None, eval_csv_path: str | None = None, *, + config: str | None = None, max_array_len: int = 128, limit: int | None = None, verbose: bool = False, @@ -59,8 +48,7 @@ def schedule_evals( local: bool = False, slurm_template_var: str | None = None, ) -> None: - """ - Schedule evaluation jobs for a given set of models, tasks, and number of shots. + """Schedule evaluation jobs for a given set of models, tasks, and number of shots. Args: models: A string of comma-separated model paths or Hugging Face model identifiers. @@ -79,6 +67,7 @@ def schedule_evals( n_shot: An integer or list of integers specifying the number of shots applied to `tasks`. eval_csv_path: A path to a CSV file containing evaluation data. Warning: exclusive argument. Cannot specify `models`, `tasks`, `task_groups`, or `n_shot` when `eval_csv_path` is provided. + config: Path to a YAML config file. CLI flags override YAML values. max_array_len: The maximum number of jobs to schedule to run concurrently. Warning: this is not the number of jobs in the array job. This is determined by the environment variable `QUEUE_LIMIT`. limit: If set, limit the number of samples per task (useful for quick testing). @@ -100,378 +89,257 @@ def schedule_evals( (PARTITION, ACCOUNT, GPUS_PER_NODE). "TIME" overrides the time limit. Example: '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00","GPUS_PER_NODE":2}' """ - _setup_logging(verbose) - - if local: - if not venv_path: - raise ValueError( - "--local requires --venv_path. Provide a path to a Python virtual " - "environment with lm_eval/lighteval installed." - ) - local_output = str(Path.cwd() / "oellm-output") - os.environ.setdefault("EVAL_BASE_DIR", local_output) - os.environ.setdefault("EVAL_OUTPUT_DIR", local_output) - os.environ.setdefault("QUEUE_LIMIT", "1") - os.environ.setdefault("GPUS_PER_NODE", "1") - os.environ.setdefault("PARTITION", "local") - os.environ.setdefault("ACCOUNT", "local") - os.environ.setdefault("EVAL_CONTAINER_IMAGE", "") - os.environ.setdefault("SINGULARITY_ARGS", "") - os.environ.setdefault("HF_HOME", str(Path.home() / ".cache" / "huggingface")) - else: - _load_cluster_env() - - use_venv = venv_path is not None + from oellm.scheduler import schedule_evals as _sched + + cli_cfg = EvalConfig.from_cli_kwargs( + models=models, + tasks=tasks, + task_groups=task_groups, + n_shot=n_shot, + eval_csv_path=eval_csv_path, + max_array_len=max_array_len, + limit=limit, + verbose=verbose, + download_only=download_only, + dry_run=dry_run, + skip_checks=skip_checks, + trust_remote_code=trust_remote_code, + venv_path=venv_path, + lm_eval_include_path=lm_eval_include_path, + local=local, + slurm_template_var=slurm_template_var, + ) - if not skip_checks: - _ensure_runtime_environment( - use_venv=use_venv, - container_image=os.environ.get("EVAL_CONTAINER_IMAGE"), - venv_path=venv_path, - ) + if config: + yaml_cfg = EvalConfig.from_yaml(config) + cfg = yaml_cfg.merge(cli_cfg) + logging.info(f"Loaded config from {config} (CLI flags override)") else: - logging.info("Skipping runtime environment check (--skip-checks enabled)") - - if isinstance(models, str): - models = [m.strip() for m in models.split(",") if m.strip()] # type: ignore - - if isinstance(tasks, str): - tasks = [t.strip() for t in tasks.split(",") if t.strip()] # type: ignore - - if isinstance(n_shot, int): - n_shot = [n_shot] + cfg = cli_cfg + + cfg.validate() + _setup_logging(cfg.verbose) + + models_str: str | None = None + if cfg._model_paths(): + models_str = ",".join(cfg._model_paths()) + + _sched( + models=models_str, + tasks=",".join(cfg.tasks) if cfg.tasks else None, + task_groups=",".join(cfg.task_groups) if cfg.task_groups else None, + n_shot=cfg.n_shot, + eval_csv_path=cfg.eval_csv_path, + max_array_len=cfg.slurm.max_array_len, + limit=cfg.limit, + verbose=cfg.verbose, + download_only=cfg.download_only, + dry_run=cfg.dry_run, + skip_checks=cfg.skip_checks, + trust_remote_code=cfg.trust_remote_code, + venv_path=cfg.venv_path, + lm_eval_include_path=cfg.lm_eval_include_path, + local=cfg.local, + slurm_template_var=cfg.slurm_template_var_json, + ) - group_names: list[str] | None = None - if task_groups: - group_names = [g.strip() for g in task_groups.split(",")] - eval_jobs: list[EvaluationJob] = [] - if eval_csv_path: - if models or tasks or task_groups or n_shot: - raise ValueError( - "Cannot specify `models`, `tasks`, `task_groups`, or `n_shot` when `eval_csv_path` is provided." - ) - df = pd.read_csv(eval_csv_path) - required_cols = {"model_path", "task_path", "n_shot"} - if not required_cols.issubset(df.columns): - raise ValueError( - f"CSV file must contain the columns: {', '.join(required_cols)}" - ) +def list_tasks(*, group: str | None = None) -> None: + """List available task groups and their tasks. - if "eval_suite" not in df.columns: - df["eval_suite"] = "lm_eval" - else: - df["eval_suite"] = df["eval_suite"].fillna("lm_eval") - - eval_jobs.extend( - [ - EvaluationJob( - model_path=row["model_path"], - task_path=row["task_path"], - n_shot=row["n_shot"], - eval_suite=row["eval_suite"], - ) - for _, row in df.iterrows() - ] - ) - - elif models: - if group_names is None: - eval_jobs.extend( - [ - EvaluationJob( - model_path=model, - task_path=task, - n_shot=shot, - eval_suite="lm_eval", - ) - for model in models - for task in tasks - for shot in n_shot - ] + Args: + group: If provided, show tasks within this specific group. + """ + from rich.table import Table + + from oellm.task_groups import TaskGroup, _parse_task_groups, get_all_task_group_names + from oellm.utils import get_console + + console = get_console() + all_names = get_all_task_group_names() + + if group: + all_names = [group] + + parsed = _parse_task_groups(all_names) + + table = Table(title="Available Task Groups") + table.add_column("Group", style="bold") + table.add_column("Suite") + table.add_column("Tasks", justify="right") + table.add_column("N-shots") + table.add_column("Description") + + for name in sorted(parsed.keys()): + g = parsed[name] + if isinstance(g, TaskGroup): + n_shots_set = set() + for t in g.tasks: + for s in t.n_shots or []: + n_shots_set.add(s) + n_shots_str = ", ".join(str(s) for s in sorted(n_shots_set)) + table.add_row( + name, + g.suite, + str(len(g.tasks)), + n_shots_str, + g.description, ) else: - expanded = _expand_task_groups(group_names) - eval_jobs.extend( - [ - EvaluationJob( - model_path=model, - task_path=result.task, - n_shot=result.n_shot, - eval_suite=result.suite, - ) - for model in models - for result in expanded - ] + # SuperGroup + total_tasks = sum(len(sg.tasks) for sg in g.task_groups) + table.add_row( + name, + "mixed", + str(total_tasks), + "", + g.description, ) - expanded_eval_jobs = [] - for job in eval_jobs: - local_model_paths = _expand_local_model_paths(job.model_path) - if not local_model_paths: - expanded_eval_jobs.append(job) - else: - for path in local_model_paths: - expanded_eval_jobs.append( - EvaluationJob( - model_path=path, - task_path=job.task_path, - n_shot=job.n_shot, - eval_suite=job.eval_suite, - ) - ) - - # For lmms_eval jobs, encode the adapter class in eval_suite as "lmms_eval:". - # This makes LMMS_MODEL_TYPE completely transparent — users never set it manually. - # For contrib suites, the registry's detect_model_flags() provides the same service. - from oellm import registry as _registry # noqa: PLC0415 - - for job in expanded_eval_jobs: - if job.eval_suite == "lmms_eval": - adapter = _detect_lmms_model_type(str(job.model_path)) - job.eval_suite = f"lmms_eval:{adapter}" - logging.debug(f"lmms-eval adapter for {job.model_path}: {adapter}") - else: - try: - mod = _registry.get_suite(job.eval_suite) - if hasattr(mod, "detect_model_flags"): - flags = mod.detect_model_flags(str(job.model_path)) - if flags: - job.eval_suite = f"{job.eval_suite}:{flags}" - logging.debug( - f"Contrib suite flags for {job.model_path} ({mod.SUITE_NAME}): {flags}" - ) - except KeyError: - pass # Not a registered contrib suite — pass eval_suite through unchanged - - if not skip_checks: - hub_models: set[str | Path] = { - job.model_path - for job in expanded_eval_jobs - if not Path(job.model_path).exists() - } - _process_model_paths(hub_models) - else: - logging.info( - "Skipping model path processing and validation (--skip-checks enabled)" - ) - - df = pd.DataFrame(expanded_eval_jobs) + console.print(table) - if df.empty: - logging.warning("No evaluation jobs to schedule.") - return None - df["eval_suite"] = df["eval_suite"].str.lower() +def compare( + result_a: str, + result_b: str, + *, + verbose: bool = False, +) -> None: + """Compare two evaluation result files or directories. - # Ensure that all datasets required by the tasks are cached locally to avoid - # network access on compute nodes. - if not skip_checks: - dataset_specs = [] - if group_names: - dataset_specs = _collect_dataset_specs(group_names) - else: - # Look up individual tasks in task groups registry - all_tasks = df["task_path"].unique().tolist() - dataset_specs = _lookup_dataset_specs_for_tasks(all_tasks) - if not dataset_specs: - logging.info( - "No dataset specs found for tasks; skipping dataset pre-download" - ) - - if dataset_specs: - _pre_download_datasets_from_specs( - dataset_specs, trust_remote_code=trust_remote_code - ) + Args: + result_a: Path to first results JSON file or directory containing results.json + result_b: Path to second results JSON file or directory containing results.json + verbose: Enable verbose logging + """ + import json - hf_model_repos = [] - if group_names: - hf_model_repos = _collect_hf_model_repos(group_names) - if hf_model_repos: - _pre_download_hf_model_repos(hf_model_repos) - - hf_dataset_files = [] - if group_names: - hf_dataset_files = _collect_hf_dataset_files(group_names) - if hf_dataset_files: - _pre_download_hf_dataset_files(hf_dataset_files) - else: - logging.info("Skipping dataset pre-download (--skip-checks enabled)") + from rich.table import Table - if download_only: - return None + from oellm.utils import get_console - remaining_queue_capacity = ( - 1 if local else int(os.environ.get("QUEUE_LIMIT", 250)) - _num_jobs_in_queue() - ) + _setup_logging(verbose) - if remaining_queue_capacity <= 0 and not dry_run: - logging.warning("No remaining queue capacity. Not scheduling any jobs.") - return None + def _load_results(path_str: str) -> list[dict]: + p = Path(path_str) + if p.is_dir(): + p = p / "results.json" + if not p.exists(): + raise FileNotFoundError(f"Results file not found: {p}") + data = json.loads(p.read_text()) + return data.get("results", []) + + results_a = _load_results(result_a) + results_b = _load_results(result_b) + + # Index by (task, n_shot, metric) + def _index(results: list[dict]) -> dict[tuple, float]: + idx = {} + for r in results: + key = (r.get("task", ""), r.get("n_shot", 0), r.get("metric", "")) + idx[key] = r.get("performance", 0.0) + return idx + + idx_a = _index(results_a) + idx_b = _index(results_b) + all_keys = sorted(set(idx_a.keys()) | set(idx_b.keys())) + + console = get_console() + table = Table(title="Comparison") + table.add_column("Task", style="bold") + table.add_column("N-shot", justify="right") + table.add_column("Metric") + table.add_column("A", justify="right") + table.add_column("B", justify="right") + table.add_column("\u0394", justify="right") # Delta + + for task, n_shot, metric in all_keys: + val_a = idx_a.get((task, n_shot, metric)) + val_b = idx_b.get((task, n_shot, metric)) + str_a = f"{val_a:.4f}" if val_a is not None else "\u2014" + str_b = f"{val_b:.4f}" if val_b is not None else "\u2014" + if val_a is not None and val_b is not None: + delta = val_b - val_a + str_delta = f"{delta:+.4f}" + else: + str_delta = "\u2014" + table.add_row(task, str(n_shot), metric, str_a, str_b, str_delta) - logging.debug( - f"Remaining capacity in the queue: {remaining_queue_capacity}. Number of " - f"evals to schedule: {len(df)}." - ) + console.print(table) - # Build a descriptive directory name: {models}_{task_groups}_{timestamp} - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - model_names = "+".join(m.split("/")[-1].lower() for m in (models or [])) - group_label = "+".join(g.lower() for g in (group_names or [])) - parts = [p for p in [model_names, group_label, timestamp] if p] - evals_dir = Path(os.environ["EVAL_OUTPUT_DIR"]) / "_".join(parts) - evals_dir.mkdir(parents=True, exist_ok=True) - - slurm_logs_dir = evals_dir / "slurm_logs" - slurm_logs_dir.mkdir(parents=True, exist_ok=True) - csv_path = evals_dir / "jobs.csv" - - # Shuffle the dataframe to distribute fast/slow evaluations evenly across array jobs - df = df.sample(frac=1, random_state=42).reset_index(drop=True) - logging.info( - "Shuffled evaluation jobs for even load distribution across array workers" - ) - df.to_csv(csv_path, index=False) +def eval_command( + config: str | None = None, + *, + models: str | None = None, + tasks: str | None = None, + task_groups: str | None = None, + n_shot: list[int] | None = None, + eval_csv_path: str | None = None, + max_array_len: int = 128, + limit: int | None = None, + verbose: bool = False, + download_only: bool = False, + dry_run: bool = False, + skip_checks: bool = False, + trust_remote_code: bool = True, + venv_path: str | None = None, + lm_eval_include_path: str | None = None, + local: bool = False, + slurm_template_var: str | None = None, +) -> None: + """Run evaluations from a YAML config file. - sbatch_template = (files("oellm.resources") / "template.sbatch").read_text() + All CLI flags override values in --config. Delegates to schedule-eval. - total_evals = len(df) - actual_array_size = min(remaining_queue_capacity, total_evals) - evals_per_job = max(1, int(math.ceil(total_evals / actual_array_size))) + Args: + config: Path to a YAML config file. + models: Comma-separated model paths or HF identifiers (overrides config). + tasks: Comma-separated task names (overrides config). + task_groups: Comma-separated task group names (overrides config). + n_shot: Number(s) of shots applied to tasks (overrides config). + eval_csv_path: Path to a CSV with evaluation jobs (overrides config). + max_array_len: Maximum concurrent SLURM array jobs. + limit: Limit samples per task. + download_only: Only pre-download models and datasets, then exit. + dry_run: Generate the SLURM script without submitting. + skip_checks: Skip container/model/dataset validation. + trust_remote_code: Trust remote code when downloading datasets. + venv_path: Python venv path. When set, runs in venv instead of Singularity. + lm_eval_include_path: Path to custom lm_eval task YAML definitions directory. + local: Run evaluations locally instead of submitting to SLURM. + slurm_template_var: JSON object of SLURM overrides. + verbose: Enable verbose logging. + """ + schedule_evals( + models=models, + tasks=tasks, + task_groups=task_groups, + n_shot=n_shot, + eval_csv_path=eval_csv_path, + config=config, + max_array_len=max_array_len, + limit=limit, + verbose=verbose, + download_only=download_only, + dry_run=dry_run, + skip_checks=skip_checks, + trust_remote_code=trust_remote_code, + venv_path=venv_path, + lm_eval_include_path=lm_eval_include_path, + local=local, + slurm_template_var=slurm_template_var, + ) - time_limit = os.environ.get("TIME_LIMIT", "12:00:00") - # Apply slurm_template_var overrides (JSON object) - if slurm_template_var: - try: - opts = json.loads(slurm_template_var) - except json.JSONDecodeError as e: - raise ValueError( - f"slurm_template_var must be a valid JSON object: {e}" - ) from e - if not isinstance(opts, dict): - raise ValueError( - "slurm_template_var must be a JSON object, e.g. " - '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00"}' - ) - for key, value in opts.items(): - if key.upper() == "TIME": - time_limit = str(value) - logging.info(f"Using time limit override: {time_limit}") - else: - os.environ[key] = str(value) - logging.info(f"Using slurm_template_var override: {key}={value}") - - logging.info("Evaluation planning:") - logging.info(f" Total evaluations: {total_evals}") - logging.info( - f" Array size: {actual_array_size} (queue capacity: {remaining_queue_capacity})" - ) - logging.info(f" Evaluations per job: {evals_per_job}") - logging.info(f" Time limit: {time_limit}") +# Register CLI commands +app.command("schedule-eval")(schedule_evals) +app.command("eval")(eval_command) +app.command("collect-results")(collect_results) +app.command("list-tasks")(list_tasks) +app.command("compare")(compare) - sbatch_script = sbatch_template.format( - csv_path=csv_path, - max_array_len=max_array_len, - array_limit=actual_array_size - 1, # Array is 0-indexed - num_jobs=actual_array_size, # This is the number of array jobs, not total evals - total_evals=len(df), # Pass the total number of evaluations - log_dir=evals_dir / "slurm_logs", - evals_dir=str(evals_dir / "results"), - time_limit=time_limit, # Dynamic time limit - limit=limit if limit else "", # Sample limit for quick testing - venv_path=venv_path or "", - lm_eval_include_path=lm_eval_include_path - or str(files("oellm.resources") / "custom_lm_eval_tasks"), - hf_hub_offline=0 if local else 1, - lighteval_model_args="trust_remote_code=True,batch_size=1" - if local - else "trust_remote_code=True", - evalchemy_dir=os.environ.get("EVALCHEMY_DIR", "/opt/evalchemy"), - ) - # substitute any $ENV_VAR occurrences - sbatch_script = Template(sbatch_script).safe_substitute(os.environ) - - sbatch_script_path = evals_dir / "submit_evals.sbatch" - - with open(sbatch_script_path, "w") as f: - f.write(sbatch_script) - - if dry_run: - logging.info(f"Dry run mode: script generated at {sbatch_script_path}") - logging.info( - f"Would run {actual_array_size} array job(s) covering {len(df)} evaluations" - ) - logging.info( - f"Each job handles ~{(len(df) + actual_array_size - 1) // actual_array_size} evaluations" - ) - if local: - logging.info( - f"To run locally: SLURM_ARRAY_TASK_ID=0 SLURM_ARRAY_JOB_ID=0 " - f"SLURM_JOB_ID=0 bash {sbatch_script_path}" - ) - else: - logging.info("To submit the job, run: sbatch " + str(sbatch_script_path)) - return - - logging.info(f"📁 Evaluation directory: {evals_dir}") - logging.info(f"📄 Script: {sbatch_script_path}") - logging.info(f"📋 Job configuration: {csv_path}") - logging.info(f"📊 Results will be stored in: {evals_dir / 'results'}") - - if local: - logging.info("Running evaluations locally with bash...") - local_env = { - **os.environ, - "SLURM_ARRAY_TASK_ID": "0", - "SLURM_ARRAY_JOB_ID": "0", - "SLURM_JOB_ID": "0", - } - try: - subprocess.run(["bash", str(sbatch_script_path)], env=local_env, check=True) - logging.info("Local evaluation completed.") - except subprocess.CalledProcessError as e: - logging.error(f"Evaluation failed with exit code {e.returncode}") - return - - try: - logging.info("Calling sbatch to launch the evaluations") - logging.info(f"📜 SLURM logs will be stored in: {slurm_logs_dir}") - - result = subprocess.run( - ["sbatch"], - input=sbatch_script, - text=True, - check=True, - capture_output=True, - env=os.environ, - ) - logging.info("Job submitted successfully.") - logging.info(result.stdout) - job_id_match = re.search(r"Submitted batch job (\d+)", result.stdout) - if job_id_match: - job_id = job_id_match.group(1) - logging.info(f"Monitor job status: squeue -j {job_id}") - logging.info(f"View job details: scontrol show job {job_id}") - logging.info(f"Cancel job if needed: scancel {job_id}") - except subprocess.CalledProcessError as e: - logging.error(f"Failed to submit job: {e}") - logging.error(f"sbatch stderr: {e.stderr}") - except FileNotFoundError: - logging.error( - "sbatch command not found. Please make sure you are on a system with SLURM installed." - ) - - -def main(): +def main() -> None: _filter_warnings() - auto_cli( - { - "schedule-eval": schedule_evals, - "collect-results": collect_results, - }, - as_positional=False, - description="OELLM: Multi-cluster evaluation tool for language models", - ) + app() diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 65a53913..6f1d757c 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -333,7 +333,7 @@ task_groups: - task: mmmu_val dataset: MMMU/MMMU - task: chartqa - dataset: HuggingFaceM4/ChartQA + dataset: lmms-lab/ChartQA - task: docvqa_val dataset: eliolio/docvqa - task: textvqa_val @@ -374,7 +374,7 @@ task_groups: n_shots: [0] tasks: - task: chartqa - dataset: HuggingFaceM4/ChartQA + dataset: lmms-lab/ChartQA image-docvqa: description: "DocVQA document visual question answering via lmms-eval" diff --git a/oellm/results.py b/oellm/results.py index 429a689c..3fe04564 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -1,7 +1,10 @@ -"""Result collection and metric resolution for evaluation outputs.""" +"""Result collection, metric resolution, and structured output for evaluation outputs.""" + +from __future__ import annotations import json import logging +from datetime import UTC, datetime from importlib.resources import files from pathlib import Path @@ -316,6 +319,16 @@ def collect_results( df = pd.DataFrame(rows) df.to_csv(output_csv, index=False) logging.info(f"Results saved to {output_csv}") + + # Write structured outputs alongside the CSV. + output_stem = Path(output_csv).with_suffix("") + json_path = Path(f"{output_stem}.json") + md_path = Path(f"{output_stem}.md") + write_results_json(rows, json_path) + write_results_markdown(rows, md_path) + logging.info(f"Results JSON: {json_path}") + logging.info(f"Results Markdown: {md_path}") + logging.info(f"Extracted {len(df)} evaluation results") if verbose: @@ -379,3 +392,73 @@ def collect_results( ) if len(missing_jobs) > 5: logging.info(f" ... and {len(missing_jobs) - 5} more") + + +# --------------------------------------------------------------------------- +# Structured output: versioned JSON and Markdown report +# --------------------------------------------------------------------------- + +SCHEMA_VERSION = "1.0" + + +def write_results_json( + rows: list[dict], + output_path: str | Path, +) -> None: + """Write evaluation results as a versioned JSON file. + + The schema is:: + + { + "version": "1.0", + "generated_at": "2026-04-02T12:00:00+00:00", + "results": [ + {"model": ..., "task": ..., "n_shot": ..., "metric": ..., "performance": ...} + ] + } + """ + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + + results = [] + for row in rows: + results.append( + { + "model": row.get("model_name", ""), + "task": row.get("task", ""), + "n_shot": row.get("n_shot", 0), + "metric": row.get("metric_name", ""), + "performance": row.get("performance", 0.0), + } + ) + + envelope = { + "version": SCHEMA_VERSION, + "generated_at": datetime.now(UTC).isoformat(), + "results": results, + } + + output_path.write_text(json.dumps(envelope, indent=2)) + + +def write_results_markdown( + rows: list[dict], + output_path: str | Path, +) -> None: + """Write evaluation results as a Markdown table.""" + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + + lines = [ + "| Model | Task | N-shot | Metric | Performance |", + "|-------|------|--------|--------|-------------|", + ] + for row in rows: + model = row.get("model_name", "") + task = row.get("task", "") + n_shot = row.get("n_shot", 0) + metric = row.get("metric_name", "") + perf = row.get("performance", 0.0) + lines.append(f"| {model} | {task} | {n_shot} | {metric} | {perf:.4f} |") + + output_path.write_text("\n".join(lines) + "\n") diff --git a/oellm/runner.py b/oellm/runner.py new file mode 100644 index 00000000..f0d050e6 --- /dev/null +++ b/oellm/runner.py @@ -0,0 +1,110 @@ +"""EvalRunner — orchestration layer for eval engine routing. + +Formalises the suite-resolution logic that determines which eval engine +handles each :class:`~oellm.constants.EvaluationJob`. The design-doc +calls this *Layer 2 — EvalRunner + Engine Routers*. + +Supported engines +----------------- +* **lm-eval** (text / multilingual) +* **lighteval** (text / multilingual) +* **lmms-eval** (image / video / audio) — adapter class auto-detected +* **contrib** suites discovered by :mod:`oellm.registry` + +The runner does **not** execute jobs — execution happens inside SLURM via +``template.sbatch``. Its job is to prepare the ``eval_suite`` column +(including adapter suffixes) so the bash-side ``case`` statement can +route correctly. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +from oellm.constants import EvaluationJob, detect_lmms_model_type + + +@dataclass(frozen=True) +class EngineInfo: + """Metadata for a known eval engine.""" + + name: str + aliases: tuple[str, ...] + + +# Known built-in engines and their normalised aliases. +ENGINES: tuple[EngineInfo, ...] = ( + EngineInfo(name="lm_eval", aliases=("lm-eval", "lm-eval-harness")), + EngineInfo(name="lighteval", aliases=("light-eval",)), + EngineInfo(name="lmms_eval", aliases=("lmms-eval",)), +) + +# Build a fast lookup: alias → canonical engine name +_ALIAS_MAP: dict[str, str] = {} +for _engine in ENGINES: + _ALIAS_MAP[_engine.name] = _engine.name + for _alias in _engine.aliases: + _ALIAS_MAP[_alias] = _engine.name + + +class EvalRunner: + """Resolve eval suites and prepare jobs for SLURM scheduling. + + Usage:: + + runner = EvalRunner() + prepared = runner.prepare_jobs(expanded_eval_jobs) + """ + + def resolve_suite(self, job: EvaluationJob) -> str: + """Return the final ``eval_suite`` string for *job*. + + For ``lmms_eval`` jobs the adapter class is auto-detected and + appended as ``lmms_eval:``. For contrib suites the + registry's ``detect_model_flags()`` provides the same service. + """ + suite = job.eval_suite + canonical = _ALIAS_MAP.get(suite, suite) + + if canonical == "lmms_eval": + adapter = detect_lmms_model_type(str(job.model_path)) + resolved = f"lmms_eval:{adapter}" + logging.debug("lmms-eval adapter for %s: %s", job.model_path, adapter) + return resolved + + # Contrib suites — attempt model-flag detection via the registry. + from oellm import registry as _registry # noqa: PLC0415 + + try: + mod = _registry.get_suite(suite) + if hasattr(mod, "detect_model_flags"): + flags = mod.detect_model_flags(str(job.model_path)) + if flags: + logging.debug( + "Contrib suite flags for %s (%s): %s", + job.model_path, + mod.SUITE_NAME, + flags, + ) + return f"{suite}:{flags}" + except KeyError: + pass # Not a registered contrib suite — pass through unchanged + + return suite + + def prepare_jobs(self, jobs: list[EvaluationJob]) -> list[EvaluationJob]: + """Resolve suites for all *jobs* in-place and return them.""" + for job in jobs: + job.eval_suite = self.resolve_suite(job) + return jobs + + @staticmethod + def canonical_name(suite: str) -> str: + """Return the canonical engine name for *suite* (or *suite* itself).""" + return _ALIAS_MAP.get(suite, suite) + + @staticmethod + def known_engines() -> list[str]: + """Return canonical names of all built-in engines.""" + return [e.name for e in ENGINES] diff --git a/oellm/scheduler.py b/oellm/scheduler.py new file mode 100644 index 00000000..c06453d0 --- /dev/null +++ b/oellm/scheduler.py @@ -0,0 +1,440 @@ +import json +import logging +import math +import os +import re +import subprocess +from datetime import datetime +from importlib.resources import files +from pathlib import Path +from string import Template + +import pandas as pd + +from oellm.constants import EvaluationJob +from oellm.runner import EvalRunner +from oellm.task_groups import ( + _collect_dataset_specs, + _collect_hf_dataset_files, + _collect_hf_model_repos, + _expand_task_groups, + _lookup_dataset_specs_for_tasks, +) +from oellm.utils import ( + _ensure_runtime_environment, + _expand_local_model_paths, + _load_cluster_env, + _num_jobs_in_queue, + _pre_download_datasets_from_specs, + _pre_download_hf_dataset_files, + _pre_download_hf_model_repos, + _process_model_paths, + _setup_logging, + capture_third_party_output_from_kwarg, +) + + +@capture_third_party_output_from_kwarg("verbose") +def schedule_evals( + models: str | None = None, + tasks: str | None = None, + task_groups: str | None = None, + n_shot: int | list[int] | None = None, + eval_csv_path: str | None = None, + *, + max_array_len: int = 128, + limit: int | None = None, + verbose: bool = False, + download_only: bool = False, + dry_run: bool = False, + skip_checks: bool = False, + trust_remote_code: bool = True, + venv_path: str | None = None, + lm_eval_include_path: str | None = None, + local: bool = False, + slurm_template_var: str | None = None, +) -> None: + """ + Schedule evaluation jobs for a given set of models, tasks, and number of shots. + + Args: + models: A string of comma-separated model paths or Hugging Face model identifiers. + Warning: does not allow passing model args such as `EleutherAI/pythia-160m,revision=step100000` + since we split on commas. If you need to pass model args, use the `eval_csv_path` option. + For local paths: + - If a directory contains `.safetensors` files directly, it will be treated as a single model + - If a directory contains subdirectories with models (e.g., converted_checkpoints/), + all models in subdirectories will be automatically discovered + - For each model directory, if it has an `hf/iter_XXXXX` structure, all checkpoints will be expanded + - This allows passing a single directory containing multiple models to evaluate them all + tasks: A string of comma-separated task names (lm_eval) or paths. + Requires `n_shot` to be provided. Tasks here are assumed to be lm_eval unless otherwise handled via CSV. + task_groups: A string of comma-separated task group names defined in `task-groups.yaml`. + Each group expands into concrete (task, n_shots, suite) entries; `n_shot` is ignored for groups. + n_shot: An integer or list of integers specifying the number of shots applied to `tasks`. + eval_csv_path: A path to a CSV file containing evaluation data. + Warning: exclusive argument. Cannot specify `models`, `tasks`, `task_groups`, or `n_shot` when `eval_csv_path` is provided. + max_array_len: The maximum number of jobs to schedule to run concurrently. + Warning: this is not the number of jobs in the array job. This is determined by the environment variable `QUEUE_LIMIT`. + limit: If set, limit the number of samples per task (useful for quick testing). + Passes --limit to lm_eval and --max_samples to lighteval. + download_only: If True, only download the datasets and models and exit. + dry_run: If True, generate the SLURM script but don't submit it to the scheduler. + skip_checks: If True, skip container image, model validation, and dataset pre-download checks for faster execution. + trust_remote_code: If True, trust remote code when downloading datasets. Default is True. Workflow might fail if set to False. + venv_path: Path to a Python virtual environment. If provided, evaluations run directly using + this venv instead of inside a Singularity/Apptainer container. + lm_eval_include_path: Path to a directory containing custom lm_eval task YAML definitions. + Passed as --include_path to lm_eval. Defaults to the bundled custom_lm_eval_tasks + directory shipped with the package. + local: If True, run evaluations directly on the local machine using bash instead of + submitting to SLURM. Requires --venv_path. + slurm_template_var: JSON object of template variable overrides. Use exact env var names + (PARTITION, ACCOUNT, GPUS_PER_NODE). "TIME" overrides the time limit. + Example: '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00","GPUS_PER_NODE":2}' + """ + _setup_logging(verbose) + + if local: + if not venv_path: + raise ValueError( + "--local requires --venv_path. Provide a path to a Python virtual " + "environment with lm_eval/lighteval installed." + ) + local_output = str(Path.cwd() / "oellm-output") + os.environ.setdefault("EVAL_BASE_DIR", local_output) + os.environ.setdefault("EVAL_OUTPUT_DIR", local_output) + os.environ.setdefault("QUEUE_LIMIT", "1") + os.environ.setdefault("GPUS_PER_NODE", "1") + os.environ.setdefault("PARTITION", "local") + os.environ.setdefault("ACCOUNT", "local") + os.environ.setdefault("EVAL_CONTAINER_IMAGE", "") + os.environ.setdefault("SINGULARITY_ARGS", "") + os.environ.setdefault("HF_HOME", str(Path.home() / ".cache" / "huggingface")) + else: + _load_cluster_env() + + use_venv = venv_path is not None + + if not skip_checks: + _ensure_runtime_environment( + use_venv=use_venv, + container_image=os.environ.get("EVAL_CONTAINER_IMAGE"), + venv_path=venv_path, + ) + else: + logging.info("Skipping runtime environment check (--skip-checks enabled)") + + if isinstance(models, str): + models = [m.strip() for m in models.split(",") if m.strip()] # type: ignore + + if isinstance(tasks, str): + tasks = [t.strip() for t in tasks.split(",") if t.strip()] # type: ignore + + if isinstance(n_shot, int): + n_shot = [n_shot] + + group_names: list[str] | None = None + if task_groups: + group_names = [g.strip() for g in task_groups.split(",")] + + eval_jobs: list[EvaluationJob] = [] + if eval_csv_path: + if models or tasks or task_groups or n_shot: + raise ValueError( + "Cannot specify `models`, `tasks`, `task_groups`, or `n_shot` when `eval_csv_path` is provided." + ) + df = pd.read_csv(eval_csv_path) + required_cols = {"model_path", "task_path", "n_shot"} + if not required_cols.issubset(df.columns): + raise ValueError( + f"CSV file must contain the columns: {', '.join(required_cols)}" + ) + + if "eval_suite" not in df.columns: + df["eval_suite"] = "lm_eval" + else: + df["eval_suite"] = df["eval_suite"].fillna("lm_eval") + + eval_jobs.extend( + [ + EvaluationJob( + model_path=row["model_path"], + task_path=row["task_path"], + n_shot=row["n_shot"], + eval_suite=row["eval_suite"], + ) + for _, row in df.iterrows() + ] + ) + + elif models: + if group_names is None: + eval_jobs.extend( + [ + EvaluationJob( + model_path=model, + task_path=task, + n_shot=shot, + eval_suite="lm_eval", + ) + for model in models + for task in tasks + for shot in n_shot + ] + ) + else: + expanded = _expand_task_groups(group_names) + eval_jobs.extend( + [ + EvaluationJob( + model_path=model, + task_path=result.task, + n_shot=result.n_shot, + eval_suite=result.suite, + ) + for model in models + for result in expanded + ] + ) + + expanded_eval_jobs = [] + for job in eval_jobs: + local_model_paths = _expand_local_model_paths(job.model_path) + if not local_model_paths: + expanded_eval_jobs.append(job) + else: + for path in local_model_paths: + expanded_eval_jobs.append( + EvaluationJob( + model_path=path, + task_path=job.task_path, + n_shot=job.n_shot, + eval_suite=job.eval_suite, + ) + ) + + # Resolve eval_suite for each job: auto-detect lmms-eval adapter classes + # and contrib suite model flags so users never set them manually. + runner = EvalRunner() + runner.prepare_jobs(expanded_eval_jobs) + + if not skip_checks: + hub_models: set[str | Path] = { + job.model_path + for job in expanded_eval_jobs + if not Path(job.model_path).exists() + } + _process_model_paths(hub_models) + else: + logging.info( + "Skipping model path processing and validation (--skip-checks enabled)" + ) + + df = pd.DataFrame(expanded_eval_jobs) + + if df.empty: + logging.warning("No evaluation jobs to schedule.") + return None + + df["eval_suite"] = df["eval_suite"].str.lower() + + # Ensure that all datasets required by the tasks are cached locally to avoid + # network access on compute nodes. + if not skip_checks: + dataset_specs = [] + if group_names: + dataset_specs = _collect_dataset_specs(group_names) + else: + # Look up individual tasks in task groups registry + all_tasks = df["task_path"].unique().tolist() + dataset_specs = _lookup_dataset_specs_for_tasks(all_tasks) + if not dataset_specs: + logging.info( + "No dataset specs found for tasks; skipping dataset pre-download" + ) + + if dataset_specs: + _pre_download_datasets_from_specs( + dataset_specs, trust_remote_code=trust_remote_code + ) + + hf_model_repos = [] + if group_names: + hf_model_repos = _collect_hf_model_repos(group_names) + if hf_model_repos: + _pre_download_hf_model_repos(hf_model_repos) + + hf_dataset_files = [] + if group_names: + hf_dataset_files = _collect_hf_dataset_files(group_names) + if hf_dataset_files: + _pre_download_hf_dataset_files(hf_dataset_files) + else: + logging.info("Skipping dataset pre-download (--skip-checks enabled)") + + if download_only: + return None + + remaining_queue_capacity = ( + 1 if local else int(os.environ.get("QUEUE_LIMIT", 250)) - _num_jobs_in_queue() + ) + + if remaining_queue_capacity <= 0 and not dry_run: + logging.warning("No remaining queue capacity. Not scheduling any jobs.") + return None + + logging.debug( + f"Remaining capacity in the queue: {remaining_queue_capacity}. Number of " + f"evals to schedule: {len(df)}." + ) + + # Build a descriptive directory name: {models}_{task_groups}_{timestamp} + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + model_names = "+".join(m.split("/")[-1].lower() for m in (models or [])) + group_label = "+".join(g.lower() for g in (group_names or [])) + parts = [p for p in [model_names, group_label, timestamp] if p] + evals_dir = Path(os.environ["EVAL_OUTPUT_DIR"]) / "_".join(parts) + evals_dir.mkdir(parents=True, exist_ok=True) + + slurm_logs_dir = evals_dir / "slurm_logs" + slurm_logs_dir.mkdir(parents=True, exist_ok=True) + csv_path = evals_dir / "jobs.csv" + + # Shuffle the dataframe to distribute fast/slow evaluations evenly across array jobs + df = df.sample(frac=1, random_state=42).reset_index(drop=True) + logging.info( + "Shuffled evaluation jobs for even load distribution across array workers" + ) + + df.to_csv(csv_path, index=False) + + sbatch_template = (files("oellm.resources") / "template.sbatch").read_text() + + total_evals = len(df) + actual_array_size = min(remaining_queue_capacity, total_evals) + evals_per_job = max(1, int(math.ceil(total_evals / actual_array_size))) + + time_limit = os.environ.get("TIME_LIMIT", "12:00:00") + + # Apply slurm_template_var overrides (JSON object) + if slurm_template_var: + try: + opts = json.loads(slurm_template_var) + except json.JSONDecodeError as e: + raise ValueError( + f"slurm_template_var must be a valid JSON object: {e}" + ) from e + if not isinstance(opts, dict): + raise ValueError( + "slurm_template_var must be a JSON object, e.g. " + '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00"}' + ) + for key, value in opts.items(): + if key.upper() == "TIME": + time_limit = str(value) + logging.info(f"Using time limit override: {time_limit}") + else: + os.environ[key] = str(value) + logging.info(f"Using slurm_template_var override: {key}={value}") + + logging.info("Evaluation planning:") + logging.info(f" Total evaluations: {total_evals}") + logging.info( + f" Array size: {actual_array_size} (queue capacity: {remaining_queue_capacity})" + ) + logging.info(f" Evaluations per job: {evals_per_job}") + logging.info(f" Time limit: {time_limit}") + + sbatch_script = sbatch_template.format( + csv_path=csv_path, + max_array_len=max_array_len, + array_limit=actual_array_size - 1, # Array is 0-indexed + num_jobs=actual_array_size, # This is the number of array jobs, not total evals + total_evals=len(df), # Pass the total number of evaluations + log_dir=evals_dir / "slurm_logs", + evals_dir=str(evals_dir / "results"), + time_limit=time_limit, # Dynamic time limit + limit=limit if limit else "", # Sample limit for quick testing + venv_path=venv_path or "", + lm_eval_include_path=lm_eval_include_path + or str(files("oellm.resources") / "custom_lm_eval_tasks"), + hf_hub_offline=0 if local else 1, + lighteval_model_args="trust_remote_code=True,batch_size=1" + if local + else "trust_remote_code=True", + evalchemy_dir=os.environ.get("EVALCHEMY_DIR", "/opt/evalchemy"), + ) + + # substitute any $ENV_VAR occurrences + sbatch_script = Template(sbatch_script).safe_substitute(os.environ) + + sbatch_script_path = evals_dir / "submit_evals.sbatch" + + with open(sbatch_script_path, "w") as f: + f.write(sbatch_script) + + if dry_run: + logging.info(f"Dry run mode: script generated at {sbatch_script_path}") + logging.info( + f"Would run {actual_array_size} array job(s) covering {len(df)} evaluations" + ) + logging.info( + f"Each job handles ~{(len(df) + actual_array_size - 1) // actual_array_size} evaluations" + ) + if local: + logging.info( + f"To run locally: SLURM_ARRAY_TASK_ID=0 SLURM_ARRAY_JOB_ID=0 " + f"SLURM_JOB_ID=0 bash {sbatch_script_path}" + ) + else: + logging.info("To submit the job, run: sbatch " + str(sbatch_script_path)) + return + + logging.info(f"Evaluation directory: {evals_dir}") + logging.info(f"Script: {sbatch_script_path}") + logging.info(f"Job configuration: {csv_path}") + logging.info(f"Results will be stored in: {evals_dir / 'results'}") + + if local: + logging.info("Running evaluations locally with bash...") + local_env = { + **os.environ, + "SLURM_ARRAY_TASK_ID": "0", + "SLURM_ARRAY_JOB_ID": "0", + "SLURM_JOB_ID": "0", + } + try: + subprocess.run(["bash", str(sbatch_script_path)], env=local_env, check=True) + logging.info("Local evaluation completed.") + except subprocess.CalledProcessError as e: + logging.error(f"Evaluation failed with exit code {e.returncode}") + return + + try: + logging.info("Calling sbatch to launch the evaluations") + logging.info(f"SLURM logs: {slurm_logs_dir}") + + result = subprocess.run( + ["sbatch"], + input=sbatch_script, + text=True, + check=True, + capture_output=True, + env=os.environ, + ) + logging.info("Job submitted successfully.") + logging.info(result.stdout) + job_id_match = re.search(r"Submitted batch job (\d+)", result.stdout) + if job_id_match: + job_id = job_id_match.group(1) + logging.info(f"Monitor job status: squeue -j {job_id}") + logging.info(f"View job details: scontrol show job {job_id}") + logging.info(f"Cancel job if needed: scancel {job_id}") + except subprocess.CalledProcessError as e: + logging.error(f"Failed to submit job: {e}") + logging.error(f"sbatch stderr: {e.stderr}") + except FileNotFoundError: + logging.error( + "sbatch command not found. Please make sure you are on a system with SLURM installed." + ) diff --git a/pyproject.toml b/pyproject.toml index b09782c5..242f8ea2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.12,<3.13" dependencies = [ "pandas", - "jsonargparse", + "typer", "datasets<4.0.0", "rich", "huggingface_hub", diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 58d736c2..c7981a24 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -250,3 +250,57 @@ def test_n_shot_zero_preserved(self, tmp_path): } df = run_collect(tmp_path, data) assert df.iloc[0]["n_shot"] == 0 + + +# ── Structured output (JSON + Markdown alongside CSV) ────────────────────── + + +class TestCollectResultsStructuredOutput: + def test_json_file_written_alongside_csv(self, tmp_path): + results_dir = tmp_path / "results" + results_dir.mkdir() + data = { + "model_name": "/path/to/model", + "results": {"copa": {"acc,none": 0.80}}, + "n-shot": {"copa": 0}, + } + write_result(results_dir, data) + output_csv = str(tmp_path / "out.csv") + collect_results(str(results_dir), output_csv=output_csv) + + json_path = tmp_path / "out.json" + assert json_path.exists() + envelope = json.loads(json_path.read_text()) + assert envelope["version"] == "1.0" + assert len(envelope["results"]) == 1 + assert envelope["results"][0]["task"] == "copa" + + def test_markdown_file_written_alongside_csv(self, tmp_path): + results_dir = tmp_path / "results" + results_dir.mkdir() + data = { + "model_name": "/path/to/model", + "results": {"copa": {"acc,none": 0.80}}, + "n-shot": {"copa": 0}, + } + write_result(results_dir, data) + output_csv = str(tmp_path / "out.csv") + collect_results(str(results_dir), output_csv=output_csv) + + md_path = tmp_path / "out.md" + assert md_path.exists() + content = md_path.read_text() + assert "copa" in content + assert "Model" in content + + def test_no_structured_output_when_no_results(self, tmp_path): + results_dir = tmp_path / "results" + results_dir.mkdir() + # Write a JSON with no extractable metrics + data = {"model_name": "m", "results": {}} + write_result(results_dir, data) + output_csv = str(tmp_path / "out.csv") + collect_results(str(results_dir), output_csv=output_csv) + + assert not (tmp_path / "out.json").exists() + assert not (tmp_path / "out.md").exists() diff --git a/tests/test_compare.py b/tests/test_compare.py new file mode 100644 index 00000000..54fa5612 --- /dev/null +++ b/tests/test_compare.py @@ -0,0 +1,206 @@ +"""Tests for the compare() CLI subcommand.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from unittest.mock import patch + +import pytest +from rich.console import Console + +from oellm.main import compare + + +def _make_results_json(path: Path, results: list[dict]) -> Path: + payload = { + "version": "1.0", + "generated_at": "2026-01-01T00:00:00+00:00", + "results": results, + } + path.write_text(json.dumps(payload)) + return path + + +def _capture_compare(result_a: str, result_b: str, **kwargs) -> str: + buf = io.StringIO() + console = Console(file=buf, highlight=False, markup=False) + with patch("oellm.utils._RICH_CONSOLE", console): + compare(result_a, result_b, **kwargs) + return buf.getvalue() + + +def test_compare_two_json_files(tmp_path: Path) -> None: + a = _make_results_json( + tmp_path / "a.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.70, + }, + ], + ) + b = _make_results_json( + tmp_path / "b.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.75, + }, + ], + ) + # Should not raise + _capture_compare(str(a), str(b)) + + +def test_compare_shows_task(tmp_path: Path) -> None: + a = _make_results_json( + tmp_path / "a.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.70, + }, + ], + ) + b = _make_results_json( + tmp_path / "b.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.75, + }, + ], + ) + output = _capture_compare(str(a), str(b)) + assert "mmlu" in output + + +def test_compare_shows_delta(tmp_path: Path) -> None: + a = _make_results_json( + tmp_path / "a.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.70, + }, + ], + ) + b = _make_results_json( + tmp_path / "b.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.75, + }, + ], + ) + output = _capture_compare(str(a), str(b)) + # Delta = 0.05 → "+0.0500" + assert "0.0500" in output + + +def test_compare_accepts_directory(tmp_path: Path) -> None: + dir_a = tmp_path / "run_a" + dir_b = tmp_path / "run_b" + dir_a.mkdir() + dir_b.mkdir() + _make_results_json( + dir_a / "results.json", + [ + { + "model": "m", + "task": "vqav2", + "n_shot": 0, + "metric": "vqa_score", + "performance": 0.80, + }, + ], + ) + _make_results_json( + dir_b / "results.json", + [ + { + "model": "m", + "task": "vqav2", + "n_shot": 0, + "metric": "vqa_score", + "performance": 0.82, + }, + ], + ) + output = _capture_compare(str(dir_a), str(dir_b)) + assert "vqav2" in output + + +def test_compare_missing_file_raises(tmp_path: Path) -> None: + a = _make_results_json(tmp_path / "a.json", []) + with pytest.raises(FileNotFoundError): + compare(str(a), str(tmp_path / "nonexistent.json")) + + +def test_compare_missing_dir_results_json_raises(tmp_path: Path) -> None: + dir_a = tmp_path / "run_a" + dir_a.mkdir() + # No results.json inside + a = _make_results_json(tmp_path / "a.json", []) + with pytest.raises(FileNotFoundError): + compare(str(a), str(dir_a)) + + +def test_compare_task_only_in_one_file(tmp_path: Path) -> None: + a = _make_results_json( + tmp_path / "a.json", + [ + { + "model": "m", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.70, + }, + ], + ) + b = _make_results_json( + tmp_path / "b.json", + [ + { + "model": "m", + "task": "vqav2", + "n_shot": 0, + "metric": "vqa_score", + "performance": 0.75, + }, + ], + ) + output = _capture_compare(str(a), str(b)) + assert "mmlu" in output + assert "vqav2" in output + # Em dash indicates missing value + assert "\u2014" in output + + +def test_compare_empty_results(tmp_path: Path) -> None: + a = _make_results_json(tmp_path / "a.json", []) + b = _make_results_json(tmp_path / "b.json", []) + # Should not raise, just print an empty table + _capture_compare(str(a), str(b)) diff --git a/tests/test_eval_command.py b/tests/test_eval_command.py new file mode 100644 index 00000000..704d18e4 --- /dev/null +++ b/tests/test_eval_command.py @@ -0,0 +1,217 @@ +"""Tests for the eval_command() CLI subcommand.""" + +from __future__ import annotations + +import inspect +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml + +from oellm.main import eval_command, schedule_evals + + +def _write_yaml(path: Path, data: dict) -> str: + path.write_text(yaml.dump(data)) + return str(path) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _run_eval(tmp_path, **kwargs): + """Run eval_command with standard dry-run patches.""" + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.runner.detect_lmms_model_type", return_value="llava"), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + eval_command(**kwargs) + + +# --------------------------------------------------------------------------- +# Basic invocation +# --------------------------------------------------------------------------- + + +def test_eval_without_config(tmp_path: Path) -> None: + _run_eval( + tmp_path, + models="EleutherAI/pythia-70m", + task_groups="open-sci-0.01", + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + +def test_eval_with_config_yaml(tmp_path: Path) -> None: + cfg = _write_yaml( + tmp_path / "eval.yaml", + { + "models": ["EleutherAI/pythia-70m"], + "task_groups": ["open-sci-0.01"], + }, + ) + _run_eval( + tmp_path, + config=cfg, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + +# --------------------------------------------------------------------------- +# CLI overrides YAML +# --------------------------------------------------------------------------- + + +def test_eval_cli_overrides_yaml_model(tmp_path: Path) -> None: + cfg = _write_yaml( + tmp_path / "eval.yaml", + { + "models": ["yaml-model"], + "task_groups": ["open-sci-0.01"], + }, + ) + with patch("oellm.scheduler.schedule_evals") as mock_sched: + with ( + patch("oellm.scheduler._load_cluster_env"), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + eval_command( + config=cfg, + models="cli-model", + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + assert mock_sched.called + called_models = mock_sched.call_args.kwargs.get("models") or mock_sched.call_args[ + 1 + ].get("models") + assert called_models == "cli-model" + + +# --------------------------------------------------------------------------- +# ModelConfig in YAML +# --------------------------------------------------------------------------- + + +def test_eval_model_config_path_extracted(tmp_path: Path) -> None: + cfg = _write_yaml( + tmp_path / "eval.yaml", + { + "models": [{"path": "/hpc/models/llava", "name": "LLaVA"}], + "task_groups": ["open-sci-0.01"], + }, + ) + with patch("oellm.scheduler.schedule_evals") as mock_sched: + with ( + patch("oellm.scheduler._load_cluster_env"), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + eval_command( + config=cfg, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + called_models = mock_sched.call_args.kwargs.get("models") or mock_sched.call_args[ + 1 + ].get("models") + assert called_models == "/hpc/models/llava" + + +def test_eval_mixed_model_list(tmp_path: Path) -> None: + cfg = _write_yaml( + tmp_path / "eval.yaml", + { + "models": [ + {"path": "/hpc/models/model-a", "name": "ModelA"}, + "EleutherAI/pythia-70m", + ], + "task_groups": ["open-sci-0.01"], + }, + ) + with patch("oellm.scheduler.schedule_evals") as mock_sched: + with ( + patch("oellm.scheduler._load_cluster_env"), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + eval_command( + config=cfg, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + called_models = mock_sched.call_args.kwargs.get("models") or mock_sched.call_args[ + 1 + ].get("models") + assert "/hpc/models/model-a" in called_models + assert "EleutherAI/pythia-70m" in called_models + + +# --------------------------------------------------------------------------- +# Validation / error cases +# --------------------------------------------------------------------------- + + +def test_eval_invalid_config_path_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + eval_command(config=str(tmp_path / "nonexistent.yaml")) + + +def test_eval_no_models_raises(tmp_path: Path) -> None: + cfg = _write_yaml( + tmp_path / "eval.yaml", + { + "task_groups": ["open-sci-0.01"], + }, + ) + with pytest.raises(ValueError, match="model"): + eval_command(config=cfg) + + +def test_eval_no_tasks_raises(tmp_path: Path) -> None: + cfg = _write_yaml( + tmp_path / "eval.yaml", + { + "models": ["EleutherAI/pythia-70m"], + }, + ) + with pytest.raises(ValueError, match="task"): + eval_command(config=cfg) + + +# --------------------------------------------------------------------------- +# schedule_evals signature unchanged +# --------------------------------------------------------------------------- + + +def test_schedule_evals_signature_unchanged() -> None: + expected_params = { + "models", + "tasks", + "task_groups", + "n_shot", + "eval_csv_path", + "max_array_len", + "limit", + "verbose", + "download_only", + "dry_run", + "skip_checks", + "trust_remote_code", + "venv_path", + "slurm_template_var", + } + actual_params = set(inspect.signature(schedule_evals).parameters.keys()) + assert expected_params.issubset(actual_params) diff --git a/tests/test_eval_config.py b/tests/test_eval_config.py new file mode 100644 index 00000000..67db125e --- /dev/null +++ b/tests/test_eval_config.py @@ -0,0 +1,400 @@ +"""Tests for EvalConfig: YAML loading, CLI construction, merge, and validation.""" + +import textwrap + +import pytest + +from oellm.config import EvalConfig, SlurmOverrides + +# --------------------------------------------------------------------------- +# SlurmOverrides +# --------------------------------------------------------------------------- + + +class TestSlurmOverrides: + def test_to_template_var_dict_empty(self): + s = SlurmOverrides() + assert s.to_template_var_dict() == {} + + def test_to_template_var_dict_partial(self): + s = SlurmOverrides(partition="dev-g", time_limit="02:00:00") + d = s.to_template_var_dict() + assert d == {"PARTITION": "dev-g", "TIME": "02:00:00"} + assert "ACCOUNT" not in d + assert "GPUS_PER_NODE" not in d + + def test_to_template_var_dict_full(self): + s = SlurmOverrides( + partition="gpu", account="FOO", gpus_per_node=4, time_limit="06:00:00" + ) + d = s.to_template_var_dict() + assert d == { + "PARTITION": "gpu", + "ACCOUNT": "FOO", + "GPUS_PER_NODE": "4", + "TIME": "06:00:00", + } + + +# --------------------------------------------------------------------------- +# from_yaml +# --------------------------------------------------------------------------- + + +class TestFromYaml: + def test_full_config(self, tmp_path): + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + textwrap.dedent("""\ + models: + - "llava-hf/llava-1.5-7b-hf" + - "Qwen/Qwen2-VL-7B" + task_groups: + - "image-vqa" + - "image-ocrbench" + n_shot: 0 + trust_remote_code: true + venv_path: "~/elliot-venv" + slurm: + max_array_len: 64 + partition: "gpu" + time_limit: "06:00:00" + """) + ) + cfg = EvalConfig.from_yaml(cfg_file) + + assert cfg.models == ["llava-hf/llava-1.5-7b-hf", "Qwen/Qwen2-VL-7B"] + assert cfg.task_groups == ["image-vqa", "image-ocrbench"] + assert cfg.n_shot == [0] + assert cfg.trust_remote_code is True + assert cfg.venv_path == "~/elliot-venv" + assert cfg.slurm.max_array_len == 64 + assert cfg.slurm.partition == "gpu" + assert cfg.slurm.time_limit == "06:00:00" + + def test_minimal_config(self, tmp_path): + cfg_file = tmp_path / "minimal.yaml" + cfg_file.write_text( + textwrap.dedent("""\ + models: + - "meta-llama/Llama-2-7b" + task_groups: + - "open-sci-0.01" + """) + ) + cfg = EvalConfig.from_yaml(cfg_file) + + assert cfg.models == ["meta-llama/Llama-2-7b"] + assert cfg.task_groups == ["open-sci-0.01"] + assert cfg.n_shot is None + assert cfg.venv_path is None + assert cfg.slurm.max_array_len == 128 # default + + def test_scalar_n_shot_becomes_list(self, tmp_path): + cfg_file = tmp_path / "scalar.yaml" + cfg_file.write_text( + textwrap.dedent("""\ + models: + - "some-model" + tasks: + - "hellaswag" + n_shot: 5 + """) + ) + cfg = EvalConfig.from_yaml(cfg_file) + assert cfg.n_shot == [5] + + def test_file_not_found(self): + with pytest.raises(FileNotFoundError, match="Config file not found"): + EvalConfig.from_yaml("/nonexistent/path.yaml") + + def test_empty_yaml(self, tmp_path): + cfg_file = tmp_path / "empty.yaml" + cfg_file.write_text("") + cfg = EvalConfig.from_yaml(cfg_file) + assert cfg.models is None + assert cfg.task_groups is None + + +# --------------------------------------------------------------------------- +# from_cli_kwargs +# --------------------------------------------------------------------------- + + +class TestFromCliKwargs: + def test_basic_cli(self): + cfg = EvalConfig.from_cli_kwargs( + models="model-a,model-b", + task_groups="image-vqa", + venv_path="~/venv", + ) + assert cfg.models == ["model-a", "model-b"] + assert cfg.task_groups == ["image-vqa"] + assert cfg.venv_path == "~/venv" + + def test_n_shot_int(self): + cfg = EvalConfig.from_cli_kwargs(models="m", tasks="t", n_shot=5) + assert cfg.n_shot == [5] + + def test_n_shot_list(self): + cfg = EvalConfig.from_cli_kwargs(models="m", tasks="t", n_shot=[0, 5]) + assert cfg.n_shot == [0, 5] + + def test_slurm_template_var_json(self): + cfg = EvalConfig.from_cli_kwargs( + models="m", + task_groups="g", + slurm_template_var='{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00","GPUS_PER_NODE":2}', + ) + assert cfg.slurm.partition == "dev-g" + assert cfg.slurm.account == "FOO" + assert cfg.slurm.time_limit == "02:00:00" + assert cfg.slurm.gpus_per_node == 2 + + def test_slurm_template_var_invalid_json(self): + with pytest.raises(ValueError, match="valid JSON"): + EvalConfig.from_cli_kwargs(models="m", slurm_template_var="not-json") + + def test_slurm_template_var_not_dict(self): + with pytest.raises(ValueError, match="JSON object"): + EvalConfig.from_cli_kwargs(models="m", slurm_template_var="[1, 2, 3]") + + +# --------------------------------------------------------------------------- +# merge (YAML base + CLI overrides) +# --------------------------------------------------------------------------- + + +class TestMerge: + def test_cli_overrides_yaml(self, tmp_path): + cfg_file = tmp_path / "base.yaml" + cfg_file.write_text( + textwrap.dedent("""\ + models: + - "yaml-model" + task_groups: + - "image-vqa" + venv_path: "~/yaml-venv" + limit: 100 + slurm: + partition: "yaml-partition" + time_limit: "10:00:00" + """) + ) + yaml_cfg = EvalConfig.from_yaml(cfg_file) + cli_cfg = EvalConfig.from_cli_kwargs( + models="cli-model", + venv_path="~/cli-venv", + ) + + merged = yaml_cfg.merge(cli_cfg) + + # CLI wins for models and venv_path + assert merged.models == ["cli-model"] + assert merged.venv_path == "~/cli-venv" + # YAML preserved for task_groups, limit, slurm + assert merged.task_groups == ["image-vqa"] + assert merged.limit == 100 + assert merged.slurm.partition == "yaml-partition" + assert merged.slurm.time_limit == "10:00:00" + + def test_cli_slurm_overrides_yaml_slurm(self, tmp_path): + cfg_file = tmp_path / "base.yaml" + cfg_file.write_text( + textwrap.dedent("""\ + models: + - "m" + task_groups: + - "g" + slurm: + partition: "yaml-part" + account: "yaml-acct" + time_limit: "10:00:00" + """) + ) + yaml_cfg = EvalConfig.from_yaml(cfg_file) + cli_cfg = EvalConfig.from_cli_kwargs( + slurm_template_var='{"PARTITION":"cli-part"}', + ) + + merged = yaml_cfg.merge(cli_cfg) + + # CLI overrides partition, YAML keeps account and time_limit + assert merged.slurm.partition == "cli-part" + assert merged.slurm.account == "yaml-acct" + assert merged.slurm.time_limit == "10:00:00" + + def test_no_cli_uses_yaml_entirely(self, tmp_path): + cfg_file = tmp_path / "base.yaml" + cfg_file.write_text( + textwrap.dedent("""\ + models: + - "yaml-model" + task_groups: + - "g" + limit: 50 + """) + ) + yaml_cfg = EvalConfig.from_yaml(cfg_file) + cli_cfg = EvalConfig.from_cli_kwargs() # all defaults + + merged = yaml_cfg.merge(cli_cfg) + + assert merged.models == ["yaml-model"] + assert merged.task_groups == ["g"] + assert merged.limit == 50 + + +# --------------------------------------------------------------------------- +# validate +# --------------------------------------------------------------------------- + + +class TestValidate: + def test_valid_task_groups(self): + cfg = EvalConfig( + models=["m"], + task_groups=["g"], + ) + cfg.validate() # should not raise + + def test_valid_tasks_with_n_shot(self): + cfg = EvalConfig( + models=["m"], + tasks=["t"], + n_shot=[0], + ) + cfg.validate() # should not raise + + def test_valid_csv_mode(self, tmp_path): + csv = tmp_path / "jobs.csv" + csv.write_text("model_path,task_path,n_shot\nm,t,0\n") + cfg = EvalConfig(eval_csv_path=str(csv)) + cfg.validate() # should not raise + + def test_no_models(self): + cfg = EvalConfig(task_groups=["g"]) + with pytest.raises(ValueError, match="At least one model"): + cfg.validate() + + def test_no_tasks_or_groups(self): + cfg = EvalConfig(models=["m"]) + with pytest.raises(ValueError, match="task_groups or tasks"): + cfg.validate() + + def test_tasks_without_n_shot(self): + cfg = EvalConfig(models=["m"], tasks=["t"]) + with pytest.raises(ValueError, match="n_shot is required"): + cfg.validate() + + def test_negative_n_shot(self): + cfg = EvalConfig(models=["m"], tasks=["t"], n_shot=[-1]) + with pytest.raises(ValueError, match="non-negative"): + cfg.validate() + + def test_csv_with_models_conflicts(self, tmp_path): + csv = tmp_path / "jobs.csv" + csv.write_text("model_path,task_path,n_shot\nm,t,0\n") + cfg = EvalConfig(eval_csv_path=str(csv), models=["m"]) + with pytest.raises(ValueError, match="Cannot specify"): + cfg.validate() + + def test_csv_not_found(self): + cfg = EvalConfig(eval_csv_path="/nonexistent/jobs.csv") + with pytest.raises(FileNotFoundError, match="eval_csv_path"): + cfg.validate() + + +# --------------------------------------------------------------------------- +# slurm_template_var_json +# --------------------------------------------------------------------------- + + +class TestSlurmTemplateVarJson: + def test_empty_returns_none(self): + cfg = EvalConfig(models=["m"], task_groups=["g"]) + assert cfg.slurm_template_var_json is None + + def test_with_overrides(self): + cfg = EvalConfig( + models=["m"], + task_groups=["g"], + slurm=SlurmOverrides(partition="dev-g", time_limit="02:00:00"), + ) + import json + + result = json.loads(cfg.slurm_template_var_json) + assert result == {"PARTITION": "dev-g", "TIME": "02:00:00"} + + +# --------------------------------------------------------------------------- +# ModelConfig +# --------------------------------------------------------------------------- + + +class TestModelConfig: + def test_modelconfig_defaults(self): + from oellm.config import ModelConfig + + m = ModelConfig(path="/some/model") + assert m.path == "/some/model" + assert m.name is None + + def test_yaml_plain_strings_backward_compat(self, tmp_path): + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + "models:\n - 'EleutherAI/pythia-70m'\ntask_groups:\n - 'open-sci-0.01'\n" + ) + cfg = EvalConfig.from_yaml(str(cfg_file)) + assert cfg.models == ["EleutherAI/pythia-70m"] + + def test_yaml_named_model_config(self, tmp_path): + from oellm.config import ModelConfig + + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + "models:\n - path: '/hpc/models/llava'\n name: 'LLaVA-1.5'\n" + "task_groups:\n - 'open-sci-0.01'\n" + ) + cfg = EvalConfig.from_yaml(str(cfg_file)) + assert len(cfg.models) == 1 + m = cfg.models[0] + assert isinstance(m, ModelConfig) + assert m.path == "/hpc/models/llava" + assert m.name == "LLaVA-1.5" + + def test_yaml_mixed_models_list(self, tmp_path): + from oellm.config import ModelConfig + + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + "models:\n - path: '/hpc/models/llava'\n name: 'LLaVA'\n" + " - 'EleutherAI/pythia-70m'\n" + "task_groups:\n - 'open-sci-0.01'\n" + ) + cfg = EvalConfig.from_yaml(str(cfg_file)) + assert len(cfg.models) == 2 + assert isinstance(cfg.models[0], ModelConfig) + assert cfg.models[1] == "EleutherAI/pythia-70m" + + def test_model_paths_helper(self, tmp_path): + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + "models:\n - path: '/hpc/models/llava'\n - 'EleutherAI/pythia-70m'\n" + "task_groups:\n - 'open-sci-0.01'\n" + ) + cfg = EvalConfig.from_yaml(str(cfg_file)) + paths = cfg._model_paths() + assert paths == ["/hpc/models/llava", "EleutherAI/pythia-70m"] + + def test_cli_kwargs_models_still_str(self): + cfg = EvalConfig.from_cli_kwargs(models="a,b", task_groups="open-sci-0.01") + assert cfg.models == ["a", "b"] + + def test_ensure_model_list_missing_path_key(self, tmp_path): + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + "models:\n - name: 'no-path'\ntask_groups:\n - 'open-sci-0.01'\n" + ) + with pytest.raises(KeyError): + EvalConfig.from_yaml(str(cfg_file)) diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py index 1e90b57b..911015c7 100644 --- a/tests/test_image_task_groups.py +++ b/tests/test_image_task_groups.py @@ -30,7 +30,7 @@ "HuggingFaceM4/VQAv2", "lmms-lab/MMBench", "MMMU/MMMU", - "HuggingFaceM4/ChartQA", + "lmms-lab/ChartQA", "eliolio/docvqa", "facebook/textvqa", "echo840/OCRBench", @@ -106,9 +106,9 @@ def test_schedule_evals_dry_run_image_vqa(self, tmp_path): from oellm.main import schedule_evals with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), - patch("oellm.main._detect_lmms_model_type", return_value="llava"), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.runner.detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( @@ -131,9 +131,9 @@ def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): from oellm.main import schedule_evals with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), - patch("oellm.main._detect_lmms_model_type", return_value="llava"), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.runner.detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( diff --git a/tests/test_list_tasks.py b/tests/test_list_tasks.py new file mode 100644 index 00000000..a7c14ef5 --- /dev/null +++ b/tests/test_list_tasks.py @@ -0,0 +1,55 @@ +"""Tests for the list_tasks() CLI subcommand.""" + +from __future__ import annotations + +import io +from unittest.mock import patch + +from rich.console import Console + +from oellm.main import list_tasks + + +def _capture_list_tasks(**kwargs) -> str: + """Run list_tasks() and return printed output as a string.""" + buf = io.StringIO() + console = Console(file=buf, highlight=False, markup=False) + with patch("oellm.utils._RICH_CONSOLE", console): + list_tasks(**kwargs) + return buf.getvalue() + + +def test_list_tasks_runs_without_error() -> None: + # Should not raise + _capture_list_tasks() + + +def test_list_tasks_includes_core_group() -> None: + output = _capture_list_tasks() + assert "open-sci-0.01" in output + + +def test_list_tasks_includes_image_group() -> None: + output = _capture_list_tasks() + assert "image-vqa" in output + + +def test_list_tasks_includes_contrib_group() -> None: + output = _capture_list_tasks() + # regiondial_bench or regiondial-bench must appear + assert "regiondial" in output.lower() + + +def test_list_tasks_has_suite_column() -> None: + output = _capture_list_tasks() + assert "Suite" in output + + +def test_list_tasks_has_nshots_column() -> None: + output = _capture_list_tasks() + assert "N-shots" in output + + +def test_list_tasks_has_tasks_column() -> None: + output = _capture_list_tasks() + assert "Tasks" in output diff --git a/tests/test_regiondial_bench.py b/tests/test_regiondial_bench.py index 49a2afc0..54662d98 100644 --- a/tests/test_regiondial_bench.py +++ b/tests/test_regiondial_bench.py @@ -503,8 +503,8 @@ def test_schedule_evals_dry_run(self, tmp_path): from oellm.main import schedule_evals with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( @@ -526,8 +526,8 @@ def test_jobs_csv_has_regiondial_bench_suite(self, tmp_path): from oellm.main import schedule_evals with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( diff --git a/tests/test_reporter.py b/tests/test_reporter.py new file mode 100644 index 00000000..57a21769 --- /dev/null +++ b/tests/test_reporter.py @@ -0,0 +1,117 @@ +"""Tests for oellm/reporter.py.""" + +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path + +import pytest + +from oellm.results import SCHEMA_VERSION, write_results_json, write_results_markdown + +_SAMPLE_ROWS = [ + { + "model_name": "/models/llava", + "task": "vqav2", + "n_shot": 0, + "performance": 0.75, + "metric_name": "vqa_score", + }, + { + "model_name": "/models/llava", + "task": "mmmu", + "n_shot": 0, + "performance": 0.55, + "metric_name": "acc", + }, +] + + +def test_write_json_schema_version(tmp_path: Path) -> None: + out = tmp_path / "results.json" + write_results_json(_SAMPLE_ROWS, out) + data = json.loads(out.read_text()) + assert data["version"] == SCHEMA_VERSION == "1.0" + + +def test_write_json_result_fields(tmp_path: Path) -> None: + out = tmp_path / "results.json" + write_results_json(_SAMPLE_ROWS, out) + data = json.loads(out.read_text()) + assert len(data["results"]) == 2 + for r in data["results"]: + assert set(r.keys()) == {"model", "task", "n_shot", "metric", "performance"} + + +def test_write_json_result_values(tmp_path: Path) -> None: + out = tmp_path / "results.json" + write_results_json(_SAMPLE_ROWS, out) + data = json.loads(out.read_text()) + first = data["results"][0] + assert first["model"] == "/models/llava" + assert first["task"] == "vqav2" + assert first["n_shot"] == 0 + assert first["metric"] == "vqa_score" + assert first["performance"] == pytest.approx(0.75) + + +def test_write_json_generated_at_is_iso8601(tmp_path: Path) -> None: + out = tmp_path / "results.json" + write_results_json(_SAMPLE_ROWS, out) + data = json.loads(out.read_text()) + # Should parse without error + dt = datetime.fromisoformat(data["generated_at"]) + assert dt.tzinfo is not None # timezone-aware + + +def test_write_json_empty_rows(tmp_path: Path) -> None: + out = tmp_path / "results.json" + write_results_json([], out) + data = json.loads(out.read_text()) + assert data["results"] == [] + assert data["version"] == "1.0" + + +def test_write_json_creates_parent_dirs(tmp_path: Path) -> None: + out = tmp_path / "subdir" / "nested" / "results.json" + assert not out.parent.exists() + write_results_json(_SAMPLE_ROWS, out) + assert out.exists() + + +def test_write_json_missing_metric_name(tmp_path: Path) -> None: + rows = [{"model_name": "m", "task": "t", "n_shot": 0, "performance": 0.5}] + out = tmp_path / "results.json" + write_results_json(rows, out) + data = json.loads(out.read_text()) + assert data["results"][0]["metric"] == "" + + +def test_write_markdown_header(tmp_path: Path) -> None: + out = tmp_path / "results.md" + write_results_markdown(_SAMPLE_ROWS, out) + text = out.read_text() + assert "| Model | Task | N-shot | Metric | Performance |" in text + assert "|-------|------|--------|--------|-------------|" in text + + +def test_write_markdown_data_row(tmp_path: Path) -> None: + out = tmp_path / "results.md" + write_results_markdown(_SAMPLE_ROWS, out) + text = out.read_text() + assert "0.7500" in text + assert "vqav2" in text + + +def test_write_markdown_creates_parent_dirs(tmp_path: Path) -> None: + out = tmp_path / "reports" / "results.md" + write_results_markdown(_SAMPLE_ROWS, out) + assert out.exists() + + +def test_write_markdown_empty_rows(tmp_path: Path) -> None: + out = tmp_path / "results.md" + write_results_markdown([], out) + text = out.read_text() + assert "| Model |" in text # header still present diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 00000000..59ceaf16 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,185 @@ +"""Tests for the EvalRunner orchestration layer.""" + +from __future__ import annotations + +from unittest.mock import patch + +from oellm.constants import EvaluationJob +from oellm.runner import _ALIAS_MAP, EvalRunner + + +class TestCanonicalName: + def test_lm_eval_canonical(self): + assert EvalRunner.canonical_name("lm_eval") == "lm_eval" + + def test_lm_eval_alias(self): + assert EvalRunner.canonical_name("lm-eval") == "lm_eval" + + def test_lm_eval_harness_alias(self): + assert EvalRunner.canonical_name("lm-eval-harness") == "lm_eval" + + def test_lighteval_canonical(self): + assert EvalRunner.canonical_name("lighteval") == "lighteval" + + def test_lighteval_alias(self): + assert EvalRunner.canonical_name("light-eval") == "lighteval" + + def test_lmms_eval_canonical(self): + assert EvalRunner.canonical_name("lmms_eval") == "lmms_eval" + + def test_lmms_eval_alias(self): + assert EvalRunner.canonical_name("lmms-eval") == "lmms_eval" + + def test_unknown_suite_passes_through(self): + assert EvalRunner.canonical_name("my_custom_suite") == "my_custom_suite" + + +class TestKnownEngines: + def test_includes_lm_eval(self): + assert "lm_eval" in EvalRunner.known_engines() + + def test_includes_lighteval(self): + assert "lighteval" in EvalRunner.known_engines() + + def test_includes_lmms_eval(self): + assert "lmms_eval" in EvalRunner.known_engines() + + def test_returns_list_of_strings(self): + engines = EvalRunner.known_engines() + assert isinstance(engines, list) + assert all(isinstance(e, str) for e in engines) + + +class TestResolveSuiteLmmsEval: + def test_lmms_eval_detects_adapter(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="llava-hf/llava-1.5-7b-hf", + task_path="vqav2_val_all", + n_shot=0, + eval_suite="lmms_eval", + ) + with patch("oellm.runner.detect_lmms_model_type", return_value="llava_hf"): + result = runner.resolve_suite(job) + assert result == "lmms_eval:llava_hf" + + def test_lmms_eval_qwen_adapter(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="Qwen/Qwen2.5-VL-7B-Instruct", + task_path="mmmu_val", + n_shot=0, + eval_suite="lmms_eval", + ) + with patch("oellm.runner.detect_lmms_model_type", return_value="qwen2_5_vl"): + result = runner.resolve_suite(job) + assert result == "lmms_eval:qwen2_5_vl" + + +class TestResolveSuiteContrib: + def test_contrib_suite_with_model_flags(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="lmsdss/RegionReasoner-7B", + task_path="regiondial_refcocog", + n_shot=0, + eval_suite="regiondial_bench", + ) + result = runner.resolve_suite(job) + assert result == "regiondial_bench:vision_reasoner" + + def test_contrib_suite_qwen_flags(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="Qwen/Qwen2.5-VL-7B-Instruct", + task_path="regiondial_refcocog", + n_shot=0, + eval_suite="regiondial_bench", + ) + result = runner.resolve_suite(job) + assert result == "regiondial_bench:qwen2.5" + + +class TestResolveSuitePassthrough: + def test_lm_eval_passes_through(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="EleutherAI/pythia-70m", + task_path="copa", + n_shot=0, + eval_suite="lm_eval", + ) + result = runner.resolve_suite(job) + assert result == "lm_eval" + + def test_lighteval_passes_through(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="EleutherAI/pythia-70m", + task_path="flores200:eng_Latn-bul_Cyrl", + n_shot=0, + eval_suite="lighteval", + ) + result = runner.resolve_suite(job) + assert result == "lighteval" + + def test_unknown_suite_passes_through(self): + runner = EvalRunner() + job = EvaluationJob( + model_path="some/model", + task_path="some_task", + n_shot=0, + eval_suite="future_engine", + ) + result = runner.resolve_suite(job) + assert result == "future_engine" + + +class TestPrepareJobs: + def test_modifies_jobs_in_place(self): + runner = EvalRunner() + jobs = [ + EvaluationJob( + model_path="EleutherAI/pythia-70m", + task_path="copa", + n_shot=0, + eval_suite="lm_eval", + ), + ] + result = runner.prepare_jobs(jobs) + assert result is jobs + assert jobs[0].eval_suite == "lm_eval" + + def test_resolves_mixed_suites(self): + runner = EvalRunner() + jobs = [ + EvaluationJob( + model_path="EleutherAI/pythia-70m", + task_path="copa", + n_shot=0, + eval_suite="lm_eval", + ), + EvaluationJob( + model_path="llava-hf/llava-1.5-7b-hf", + task_path="vqav2_val_all", + n_shot=0, + eval_suite="lmms_eval", + ), + ] + with patch("oellm.runner.detect_lmms_model_type", return_value="llava_hf"): + runner.prepare_jobs(jobs) + + assert jobs[0].eval_suite == "lm_eval" + assert jobs[1].eval_suite == "lmms_eval:llava_hf" + + def test_empty_list(self): + runner = EvalRunner() + result = runner.prepare_jobs([]) + assert result == [] + + +class TestAliasMap: + def test_all_engines_have_canonical_entry(self): + for name in EvalRunner.known_engines(): + assert name in _ALIAS_MAP + assert _ALIAS_MAP[name] == name diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index 39a2cb90..325d1c82 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -17,9 +17,9 @@ @pytest.mark.parametrize("task_groups", ALL_TASK_GROUPS) def test_schedule_evals(tmp_path, n_shot, task_groups): with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), - patch("oellm.main._detect_lmms_model_type", return_value="llava"), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.runner.detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): schedule_evals( @@ -35,8 +35,8 @@ def test_schedule_evals(tmp_path, n_shot, task_groups): def test_schedule_evals_slurm_template_var_overrides(tmp_path): """Verify --slurm_template_var JSON overrides appear in the generated sbatch.""" with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), patch.dict( os.environ, { @@ -68,8 +68,8 @@ def test_schedule_evals_slurm_template_var_overrides(tmp_path): def test_schedule_evals_slurm_template_var_invalid_json(tmp_path): """Verify invalid slurm_template_var raises ValueError.""" with ( - patch("oellm.main._load_cluster_env"), - patch("oellm.main._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): with pytest.raises(ValueError, match="valid JSON object"): From 937e3da11fe341adc74e3de5744ab398a58a203f Mon Sep 17 00:00:00 2001 From: islobozhan Date: Tue, 14 Apr 2026 12:35:42 +0200 Subject: [PATCH 18/44] [Base][New Modality] Adding support for video modality and benchmarks Add video understanding task group with 5 benchmarks via lmms-eval: 1. VideoMMMU, 2. EgoSchema, 3. VideoMME, 4. ActivityNet-QA, 5. LongVideoBench --- README.md | 20 +- docs/VENV.md | 2 +- oellm/constants.py | 7 + oellm/contrib/regiondial_bench/__init__.py | 1 - oellm/resources/task-groups.yaml | 65 +++++++ oellm/resources/template.sbatch | 10 +- oellm/task_groups.py | 59 +++--- oellm/utils.py | 45 +++-- pyproject.toml | 3 + requirements-venv-evalchemy.txt | 2 +- tests/test_video_task_groups.py | 205 +++++++++++++++++++++ 11 files changed, 371 insertions(+), 48 deletions(-) create mode 100644 tests/test_video_task_groups.py diff --git a/README.md b/README.md index 799a0af6..6214e306 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Snellius) - **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) +- **Video evaluation** via lmms-eval (MVBench, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench) - **Plugin system** for contributing custom benchmarks without touching core code - **Automatic building and deployment of containers** @@ -75,7 +76,18 @@ Super groups: `oellm-multilingual` (all multilingual benchmarks combined) | `image-ocrbench` | OCRBench | lmms-eval | | `image-mathvista` | MathVista | lmms-eval | -The lmms-eval adapter class (`llava_hf`, `qwen2_5_vl`, etc.) is auto-detected from the model name. +### Video + +| Group | Benchmark | Engine | +|---|---|---| +| `video-understanding` | All 5 benchmarks combined | lmms-eval | +| `video-mvbench` | MVBench (20 temporal tasks) | lmms-eval | +| `video-egoschema` | EgoSchema (long-form egocentric QA) | lmms-eval | +| `video-videomme` | Video-MME (11s-1h clips) | lmms-eval | +| `video-activitynet-qa` | ActivityNet-QA (requires GPT API) | lmms-eval | +| `video-longvideobench` | LongVideoBench (cross-segment reasoning) | lmms-eval | + +The lmms-eval adapter class (`llava_hf`, `llava_onevision`, `qwen2_5_vl`, etc.) is auto-detected from the model name. Install with `pip install oellm[video]` (or use a venv with lmms-eval). ### Custom Benchmarks (contrib) @@ -88,6 +100,12 @@ oellm schedule-eval \ --task-groups "image-vqa" \ --venv-path ~/elliot-venv +# Run all 5 video benchmarks +oellm schedule-eval \ + --models "lmms-lab/llava-onevision-7b" \ + --task-groups "video-understanding" \ + --venv-path ~/elliot-venv + # Mix image and text benchmarks in one submission oellm schedule-eval \ --models "llava-hf/llava-1.5-7b-hf" \ diff --git a/docs/VENV.md b/docs/VENV.md index bce8ac43..553500fc 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -95,7 +95,7 @@ We use [Ali's fork](https://github.com/Ali-Elganzory/evalchemy) which includes a 3. Run with `EVALCHEMY_DIR` pointing to the cloned repo: ```bash - export HF_ALLOW_CODE_EVAL=1 # required by MBPP + export HF_ALLOW_CODE_EVAL=1 # required by MBPP EVALCHEMY_DIR=$(pwd)/evalchemy oellm schedule-eval \ --models HuggingFaceTB/SmolLM2-135M \ --task-groups reasoning \ diff --git a/oellm/constants.py b/oellm/constants.py index 15915a5d..c8c73349 100644 --- a/oellm/constants.py +++ b/oellm/constants.py @@ -18,10 +18,17 @@ class EvaluationJob: LMMS_MODEL_ADAPTERS: list[tuple[list[str], str]] = [ (["qwen2.5-vl", "qwen2_5_vl", "qwen2.5vl"], "qwen2_5_vl"), (["qwen2-vl", "qwen2_vl"], "qwen2_vl"), + (["llava-hf"], "llava_hf"), + (["llava-onevision", "llava_onevision"], "llava_onevision"), + (["llava-vid", "llava_vid", "llava-video"], "llava_vid"), + (["video-llava", "video_llava"], "video_llava"), (["llava"], "llava_hf"), + (["internvideo"], "internvideo2"), (["internvl"], "internvl2"), (["idefics"], "idefics3"), (["minicpm"], "minicpm_v"), + (["longva"], "longva"), + (["videochat2"], "videochat2"), (["qwen"], "qwen_vl"), ] diff --git a/oellm/contrib/regiondial_bench/__init__.py b/oellm/contrib/regiondial_bench/__init__.py index 8b137891..e69de29b 100644 --- a/oellm/contrib/regiondial_bench/__init__.py +++ b/oellm/contrib/regiondial_bench/__init__.py @@ -1 +0,0 @@ - diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 6f1d757c..ba602e2a 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -35,6 +35,13 @@ task_metrics: mathvista_testmini_cot: llm_as_judge_eval mathvista_testmini_format: llm_as_judge_eval mathvista_testmini_solution: llm_as_judge_eval + # lmms-eval video benchmark metrics + video_mmmu: mmmu_acc + egoschema: submission + videomme: videomme_perception_score + # ActivityNet-QA requires GPT API access for evaluation (LLM-as-judge) + activitynetqa: gpt_eval_accuracy + longvideobench_val_v: lvb_acc task_groups: open-sci-0.01: @@ -416,6 +423,64 @@ task_groups: - task: mathvista_testmini dataset: AI4Math/MathVista + # ── Video Modality (lmms-eval) ──────────────────────────────────────────── + video-understanding: + description: "Video understanding benchmarks via lmms-eval (VideoMMMU, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: video_mmmu + dataset: lmms-lab/VideoMMMU + - task: egoschema + dataset: lmms-lab/egoschema + - task: videomme + dataset: lmms-lab/Video-MME + - task: activitynetqa + dataset: lmms-lab/ActivityNetQA + - task: longvideobench_val_v + dataset: longvideobench/LongVideoBench + + # ── Individual Video Benchmarks (single-task groups for targeted runs) ──── + video-videommmu: + description: "VideoMMMU multi-discipline video understanding via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: video_mmmu + dataset: lmms-lab/VideoMMMU + + video-egoschema: + description: "EgoSchema long-form egocentric video QA via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: egoschema + dataset: lmms-lab/egoschema + + video-videomme: + description: "Video-MME full-spectrum video understanding (11s-1h) via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: videomme + dataset: lmms-lab/Video-MME + + video-activitynet-qa: + description: "ActivityNet-QA open-ended activity video QA via lmms-eval (requires GPT API for scoring)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: activitynetqa + dataset: lmms-lab/ActivityNetQA + + video-longvideobench: + description: "LongVideoBench long-video cross-segment reasoning via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: longvideobench_val_v + dataset: longvideobench/LongVideoBench + dclm-core-22: description: "DCLM core 22 evaluation tasks (lm-eval-harness, matching LLM Foundry task types)" suite: lm-eval-harness diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 17c374b2..d67ab948 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -190,11 +190,17 @@ do _lmms_adapter="${{eval_suite#*:}}" OUTPUT_JSON="{evals_dir}/$(openssl rand -hex 5).json" + # LLaVA adapters need model_name to avoid a missing-import bug in lmms-eval + _lmms_extra_args="" + if [[ "$_lmms_adapter" == "llava_onevision" || "$_lmms_adapter" == "llava_vid" || "$_lmms_adapter" == "video_llava" ]]; then + _lmms_extra_args=",model_name=$(basename "$model_path")" + fi + if [ -n "$VENV_PATH" ]; then source "$VENV_PATH/bin/activate" python -m lmms_eval \ --model "$_lmms_adapter" \ - --model_args "pretrained=$model_path,device_map=auto" \ + --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args" \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ @@ -205,7 +211,7 @@ do $EVAL_SIF_PATH \ python -m lmms_eval \ --model "$_lmms_adapter" \ - --model_args "pretrained=$model_path,device_map=auto" \ + --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args" \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ diff --git a/oellm/task_groups.py b/oellm/task_groups.py index 72914201..e925273d 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -9,6 +9,7 @@ class DatasetSpec: repo_id: str subset: str | None = None + video: bool = False @dataclass @@ -154,16 +155,16 @@ class TaskGroupResult: def _iter_all_tasks( parsed: dict[str, TaskSuperGroup | TaskGroup], -) -> Iterable[tuple[_Task, str]]: - """Yield ``(task, suite)`` pairs from a parsed group dict, flattening super groups.""" - for group in parsed.values(): +) -> Iterable[tuple[_Task, str, str]]: + """Yield ``(task, suite, group_name)`` triples from a parsed group dict, flattening super groups.""" + for group_name, group in parsed.items(): if isinstance(group, TaskGroup): for t in group.tasks: - yield t, t.suite or group.suite + yield t, t.suite or group.suite, group_name else: for g in group.task_groups: for t in g.tasks: - yield t, t.suite or g.suite + yield t, t.suite or g.suite, g.name def _expand_task_groups(group_names: Iterable[str]) -> list[TaskGroupResult]: @@ -173,7 +174,7 @@ def _expand_task_groups(group_names: Iterable[str]) -> list[TaskGroupResult]: raise ValueError(f"Unknown task group(s): {', '.join(sorted(missing))}") results: list[TaskGroupResult] = [] - for t, suite in _iter_all_tasks(parsed): + for t, suite, _gname in _iter_all_tasks(parsed): for shot in (int(s) for s in (t.n_shots or [])): results.append(TaskGroupResult(task=t.name, n_shot=shot, suite=suite)) @@ -198,22 +199,28 @@ def _collect_dataset_specs(group_names: Iterable[str]) -> list[DatasetSpec]: parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) specs: list[DatasetSpec] = [] - seen: set[tuple[str, str | None]] = set() + seen: set[tuple[str, str | None, str | None]] = set() - def add_spec(dataset: str | None, subset: str | None): + def add_spec( + dataset: str | None, + subset: str | None, + video: bool = False, + ): if dataset is None: return key = (dataset, subset) if key not in seen: seen.add(key) - specs.append(DatasetSpec(repo_id=dataset, subset=subset)) + specs.append(DatasetSpec(repo_id=dataset, subset=subset, video=video)) + + for t, _, group_name in _iter_all_tasks(parsed): + is_video = group_name.startswith("video-") - for t, _ in _iter_all_tasks(parsed): if t.dataset == "facebook/flores" and not t.subset: for lang in _extract_flores_subsets(t.name): add_spec(t.dataset, lang) else: - add_spec(t.dataset, t.subset) + add_spec(t.dataset, t.subset, video=is_video) return specs @@ -225,7 +232,7 @@ def _collect_hf_model_repos(group_names: Iterable[str]) -> list[str]: repos: list[str] = [] seen: set[str] = set() - for t, _ in _iter_all_tasks(parsed): + for t, _, _gname in _iter_all_tasks(parsed): for repo_id in t.hf_models or []: if repo_id not in seen: seen.add(repo_id) @@ -238,24 +245,32 @@ def _collect_hf_dataset_files(group_names: Iterable[str]) -> list[dict]: """Return deduplicated HF dataset file specs declared in task ``hf_dataset_files`` fields.""" parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) - # Merge patterns from all tasks that share the same repo_id so that - # a single snapshot_download fetches everything needed. - merged: dict[str, list[str]] = {} + # Merge patterns from all tasks that share the same (repo_id, revision) + # so that a single snapshot_download fetches everything needed. + merged: dict[tuple[str, str | None], list[str]] = {} - for t, _ in _iter_all_tasks(parsed): + for t, _, _gname in _iter_all_tasks(parsed): for spec in t.hf_dataset_files or []: repo_id = spec.get("repo_id", "") if not repo_id: continue + revision = spec.get("revision") patterns = spec.get("patterns") or [] - if repo_id not in merged: - merged[repo_id] = list(patterns) + key = (repo_id, revision) + if key not in merged: + merged[key] = list(patterns) else: for p in patterns: - if p not in merged[repo_id]: - merged[repo_id].append(p) + if p not in merged[key]: + merged[key].append(p) - return [{"repo_id": rid, "patterns": pats} for rid, pats in merged.items()] + result = [] + for (rid, rev), pats in merged.items(): + entry: dict = {"repo_id": rid, "patterns": pats} + if rev: + entry["revision"] = rev + result.append(entry) + return result def _build_task_dataset_map() -> dict[str, list[DatasetSpec]]: @@ -268,7 +283,7 @@ def _build_task_dataset_map() -> dict[str, list[DatasetSpec]]: task_map: dict[str, list[DatasetSpec]] = {} - for t, _ in _iter_all_tasks(parsed): + for t, _, _gname in _iter_all_tasks(parsed): if t.dataset and t.name not in task_map: if t.dataset == "facebook/flores" and not t.subset: task_map[t.name] = [ diff --git a/oellm/utils.py b/oellm/utils.py index 27bde7d5..0c381dad 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -314,21 +314,9 @@ def _process_model_paths(models: Iterable[str]): if "HF_HOME" in os.environ else None ) - try: - from huggingface_hub import try_to_load_from_cache - - cached = try_to_load_from_cache( - model, "config.json", cache_dir=cache_dir - ) - if isinstance(cached, str): - logging.info( - f"Model '{model}' already cached, skipping download" - ) - per_model_paths.append(model) - continue - except Exception: - pass status.update(f"Downloading '{model}' ({idx}/{len(models_list)})") + # snapshot_download is idempotent — it skips files that + # are already cached and only fetches missing ones. snapshot_download( repo_id=model, cache_dir=cache_dir, @@ -373,16 +361,20 @@ def _pre_download_hf_dataset_files(dataset_files: list[dict]) -> None: for idx, spec in enumerate(dataset_files, 1): repo_id = spec.get("repo_id", "") patterns = spec.get("patterns") + revision = spec.get("revision") status.update(f"Downloading '{repo_id}' ({idx}/{len(dataset_files)})") try: - snapshot_download( - repo_id=repo_id, - repo_type="dataset", - allow_patterns=patterns, - cache_dir=Path(os.getenv("HF_HOME")) / "hub" + kwargs = { + "repo_id": repo_id, + "repo_type": "dataset", + "allow_patterns": patterns, + "cache_dir": Path(os.getenv("HF_HOME")) / "hub" if "HF_HOME" in os.environ else None, - ) + } + if revision: + kwargs["revision"] = revision + snapshot_download(**kwargs) except Exception as e: logging.warning(f"Failed to download dataset files from '{repo_id}': {e}") @@ -391,6 +383,7 @@ def _pre_download_datasets_from_specs( specs: Iterable, trust_remote_code: bool = True ) -> None: from datasets import get_dataset_config_names, load_dataset + from huggingface_hub import snapshot_download specs_list = list(specs) if not specs_list: @@ -406,6 +399,18 @@ def _pre_download_datasets_from_specs( label = f"{spec.repo_id}" + (f"/{spec.subset}" if spec.subset else "") status.update(f"Downloading '{label}' ({idx}/{len(specs_list)})") + # Video datasets: lmms-eval calls snapshot_download at runtime + # to get raw video files, then symlinks them into $HF_HOME. + # Pre-download so offline compute nodes find everything cached. + if spec.video: + try: + snapshot_download( + repo_id=spec.repo_id, + repo_type="dataset", + ) + except Exception as e: + logging.warning(f"Failed to snapshot_download '{spec.repo_id}': {e}") + try: load_dataset( spec.repo_id, diff --git a/pyproject.toml b/pyproject.toml index 242f8ea2..37a2f8f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,9 @@ dev = [ image = [ "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", ] +video = [ + "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", +] [project.scripts] oellm = "oellm.main:main" diff --git a/requirements-venv-evalchemy.txt b/requirements-venv-evalchemy.txt index 9ec7d3d9..c63c3fbc 100644 --- a/requirements-venv-evalchemy.txt +++ b/requirements-venv-evalchemy.txt @@ -1,6 +1,6 @@ # Dependencies for evalchemy evaluation -# lm-eval fork used by evalchemy +# lm-eval fork used by evalchemy lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@etashg/tokenize_fix scipy==1.17.0 diff --git a/tests/test_video_task_groups.py b/tests/test_video_task_groups.py new file mode 100644 index 00000000..f5d72741 --- /dev/null +++ b/tests/test_video_task_groups.py @@ -0,0 +1,205 @@ +import os +import sys +from importlib.resources import files +from pathlib import Path +from unittest.mock import patch + +import yaml + +from oellm.task_groups import ( + _collect_dataset_specs, + _expand_task_groups, + get_all_task_group_names, +) + +VIDEO_TASK_GROUP = "video-understanding" + +EXPECTED_TASKS = { + "video_mmmu", + "egoschema", + "videomme", + "activitynetqa", + "longvideobench_val_v", +} + +EXPECTED_DATASETS = { + "lmms-lab/VideoMMMU", + "lmms-lab/egoschema", + "lmms-lab/Video-MME", + "lmms-lab/ActivityNetQA", + "longvideobench/LongVideoBench", +} + + +class TestVideoTaskGroupInRegistry: + def test_video_understanding_present_in_yaml(self): + all_groups = get_all_task_group_names() + assert VIDEO_TASK_GROUP in all_groups + + def test_video_understanding_suite_is_lmms_eval(self): + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + suite = data["task_groups"][VIDEO_TASK_GROUP]["suite"] + assert suite == "lmms_eval" + + def test_video_understanding_has_five_tasks(self): + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + tasks = data["task_groups"][VIDEO_TASK_GROUP]["tasks"] + assert len(tasks) == 5 + + def test_individual_video_groups_present(self): + all_groups = get_all_task_group_names() + for name in [ + "video-videommmu", + "video-egoschema", + "video-videomme", + "video-activitynet-qa", + "video-longvideobench", + ]: + assert name in all_groups, f"{name} not in task group registry" + + +class TestVideoTaskGroupExpansion: + def test_expands_to_correct_task_names(self): + results = _expand_task_groups([VIDEO_TASK_GROUP]) + task_names = {r.task for r in results} + assert task_names == EXPECTED_TASKS + + def test_all_tasks_have_zero_shot(self): + results = _expand_task_groups([VIDEO_TASK_GROUP]) + for r in results: + assert r.n_shot == 0, f"{r.task} has n_shot={r.n_shot}, expected 0" + + def test_all_tasks_route_to_lmms_eval(self): + results = _expand_task_groups([VIDEO_TASK_GROUP]) + for r in results: + assert r.suite == "lmms_eval", ( + f"{r.task} has suite='{r.suite}', expected 'lmms_eval'" + ) + + def test_expand_individual_video_group(self): + results = _expand_task_groups(["video-videommmu"]) + assert len(results) == 1 + assert results[0].task == "video_mmmu" + assert results[0].suite == "lmms_eval" + + +class TestVideoTaskGroupDatasetSpecs: + def test_all_expected_datasets_present(self): + specs = _collect_dataset_specs([VIDEO_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert repo_ids == EXPECTED_DATASETS + + def test_no_duplicate_dataset_specs(self): + specs = _collect_dataset_specs([VIDEO_TASK_GROUP]) + keys = [(s.repo_id, s.subset) for s in specs] + assert len(keys) == len(set(keys)), "Duplicate dataset specs found" + + def test_videomme_dataset_included(self): + specs = _collect_dataset_specs([VIDEO_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert "lmms-lab/Video-MME" in repo_ids + + +class TestVideoTaskGroupScheduleEvals: + """Verify video-understanding integrates with the schedule_evals dry-run path.""" + + def test_schedule_evals_dry_run_video(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch( + "oellm.runner.detect_lmms_model_type", + return_value="llava_onevision", + ), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="lmms-lab/llava-onevision-7b", + task_groups=VIDEO_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + sbatch_content = sbatch_files[0].read_text() + assert "lmms_eval" in sbatch_content + + def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): + import pandas as pd + + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch( + "oellm.runner.detect_lmms_model_type", + return_value="llava_onevision", + ), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="lmms-lab/llava-onevision-7b", + task_groups=VIDEO_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + csv_files = list(tmp_path.glob("**/jobs.csv")) + assert len(csv_files) == 1 + df = pd.read_csv(csv_files[0]) + assert all(s.startswith("lmms_eval") for s in df["eval_suite"].unique()) + assert set(df["task_path"].unique()) == EXPECTED_TASKS + + +class TestVideoModelAdapters: + """Verify video-specific model adapter detection.""" + + def test_llava_onevision_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("lmms-lab/llava-onevision-7b") == "llava_onevision" + + def test_llava_vid_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("llava-vid-7b") == "llava_vid" + + def test_video_llava_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("video-llava-7b") == "video_llava" + + def test_longva_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("longva-7b") == "longva" + + def test_internvideo_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("internvideo2-chat") == "internvideo2" + + def test_llava_hf_onevision_routes_to_llava_hf(self): + """HuggingFace-format llava-onevision models must use llava_hf, not llava_onevision.""" + from oellm.constants import detect_lmms_model_type + + assert ( + detect_lmms_model_type("llava-hf/llava-onevision-qwen2-0.5b-ov-hf") + == "llava_hf" + ) + + def test_generic_llava_still_works(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("llava-hf/llava-1.5-7b-hf") == "llava_hf" + + def test_qwen25_vl_still_works(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("Qwen/Qwen2.5-VL-7B-Instruct") == "qwen2_5_vl" From f73d9dd6a3d0407ee38edc5b7cb65ced06eb8f14 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Tue, 14 Apr 2026 13:12:05 +0200 Subject: [PATCH 19/44] fix lints --- oellm/scheduler.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/oellm/scheduler.py b/oellm/scheduler.py index 98c288fe..7314a3b7 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -13,22 +13,6 @@ from oellm.constants import EvaluationJob from oellm.runner import EvalRunner - - -def _resolve_hf_hub_offline(local: bool) -> int: - """Value embedded in the generated eval script as HF_HUB_OFFLINE. - - If ``HF_HUB_OFFLINE`` is set in the environment when ``oellm`` runs, that - value wins. Otherwise defaults to online Hub access for ``--local`` - (typical laptop dev) and offline for SLURM jobs (air-gapped workers). - """ - raw = os.environ.get("HF_HUB_OFFLINE") - if raw is not None and str(raw).strip() != "": - try: - return int(str(raw).strip()) - except ValueError: - logging.warning("Invalid HF_HUB_OFFLINE=%r; using default", raw) - return 0 if local else 1 from oellm.task_groups import ( _collect_dataset_specs, _collect_hf_dataset_files, @@ -50,6 +34,22 @@ def _resolve_hf_hub_offline(local: bool) -> int: ) +def _resolve_hf_hub_offline(local: bool) -> int: + """Value embedded in the generated eval script as HF_HUB_OFFLINE. + + If ``HF_HUB_OFFLINE`` is set in the environment when ``oellm`` runs, that + value wins. Otherwise defaults to online Hub access for ``--local`` + (typical laptop dev) and offline for SLURM jobs (air-gapped workers). + """ + raw = os.environ.get("HF_HUB_OFFLINE") + if raw is not None and str(raw).strip() != "": + try: + return int(str(raw).strip()) + except ValueError: + logging.warning("Invalid HF_HUB_OFFLINE=%r; using default", raw) + return 0 if local else 1 + + @capture_third_party_output_from_kwarg("verbose") def schedule_evals( models: str | None = None, From a48c24d4d1b7f1e7f33f04b142bb6f55ab004236 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 20 Apr 2026 17:29:15 +0200 Subject: [PATCH 20/44] [Base][New Modality] Adding audio modality benchmarks from llms-eval Adding audio modality benchmarks from llms-eval --- README.md | 29 ++- oellm/constants.py | 10 + oellm/resources/task-groups.yaml | 344 ++++++++++++++++++++++++++++--- oellm/task_groups.py | 16 +- oellm/utils.py | 10 +- pyproject.toml | 7 + requirements-venv.txt | 5 + tests/test_audio_task_groups.py | 326 +++++++++++++++++++++++++++++ tests/test_collect_results.py | 174 +++++++++++++--- tests/test_image_task_groups.py | 17 +- tests/test_runner.py | 4 +- tests/test_video_task_groups.py | 32 ++- 12 files changed, 896 insertions(+), 78 deletions(-) create mode 100644 tests/test_audio_task_groups.py diff --git a/README.md b/README.md index b8c28c29..1a44218f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Snellius) - **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) -- **Video evaluation** via lmms-eval (MVBench, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench) +- **Video evaluation** via lmms-eval (VideoMMMU, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench) +- **Audio evaluation** via lmms-eval (LibriSpeech, FLEURS, GigaSpeech, TED-LIUM, WenetSpeech, CoVoST2, VocalSound, MuChoMusic) - **Plugin system** for contributing custom benchmarks without touching core code - **Automatic building and deployment of containers** @@ -74,14 +75,14 @@ Super groups: `oellm-multilingual` (all multilingual benchmarks combined) | `image-docvqa` | DocVQA | lmms-eval | | `image-textvqa` | TextVQA | lmms-eval | | `image-ocrbench` | OCRBench | lmms-eval | -| `image-mathvista` | MathVista | lmms-eval | +| `image-mathvista` | MathVista (CoT / format / solution leaves — needs GPT judge) | lmms-eval | ### Video | Group | Benchmark | Engine | |---|---|---| | `video-understanding` | All 5 benchmarks combined | lmms-eval | -| `video-mvbench` | MVBench (20 temporal tasks) | lmms-eval | +| `video-videommmu` | VideoMMMU (perception / comprehension / adaptation leaves) | lmms-eval | | `video-egoschema` | EgoSchema (long-form egocentric QA) | lmms-eval | | `video-videomme` | Video-MME (11s-1h clips) | lmms-eval | | `video-activitynet-qa` | ActivityNet-QA (requires GPT API) | lmms-eval | @@ -89,6 +90,28 @@ Super groups: `oellm-multilingual` (all multilingual benchmarks combined) The lmms-eval adapter class (`llava_hf`, `llava_onevision`, `qwen2_5_vl`, etc.) is auto-detected from the model name. Install with `pip install oellm[video]` (or use a venv with lmms-eval). +### Audio + +| Group | Benchmark | Engine | +|---|---|---| +| `audio-understanding` | Curated suite: 8 leaf tasks, no judge-model dependency | lmms-eval | +| `audio-librispeech` | LibriSpeech ASR (WER on test-clean) | lmms-eval | +| `audio-common-voice-15` | Common Voice 15 (en, fr, zh-CN) | lmms-eval | +| `audio-gigaspeech` | GigaSpeech large-scale ASR | lmms-eval | +| `audio-tedlium` | TED-LIUM v3 ASR | lmms-eval | +| `audio-wenet-speech` | WenetSpeech Chinese ASR (MER) | lmms-eval | +| `audio-covost2` | CoVoST2 en→zh speech translation (BLEU) | lmms-eval | +| `audio-fleurs` | FLEURS multilingual speech | lmms-eval | +| `audio-voxpopuli`, `audio-ami`, `audio-people-speech` | Additional ASR corpora | lmms-eval | +| `audio-vocalsound` | Non-speech vocalisation classification | lmms-eval | +| `audio-muchomusic` | Music understanding MCQ | lmms-eval | +| `audio-air-bench-chat` | AIR-Bench chat (requires GPT judge) | lmms-eval | +| `audio-air-bench-foundation` | AIR-Bench foundation MCQ | lmms-eval | +| `audio-alpaca-audio`, `audio-openhermes`, `audio-wavcaps` | Instruction / captioning (GPT judge) | lmms-eval | +| `audio-clotho-aqa`, `audio-cn-college-listen-mcq`, `audio-dream-tts-mcq`, `audio-voicebench`, `audio-step2-paralinguistic` | QA / MCQ / paralinguistic probes | lmms-eval | + +Install with `pip install oellm[audio]`. Judge-model groups (AIR-Bench chat, Alpaca-Audio, OpenHermes, WavCaps) need `OPENAI_API_KEY` on the compute node. The HPC Singularity image must include `ffmpeg` for non-WAV decode. + ### Custom Benchmarks (contrib) Community-contributed benchmarks that run outside the standard evaluation engines. See the [contrib registry](oellm/contrib/README.md) for the full list. diff --git a/oellm/constants.py b/oellm/constants.py index c8c73349..bb52e2a0 100644 --- a/oellm/constants.py +++ b/oellm/constants.py @@ -16,6 +16,16 @@ class EvaluationJob: # Patterns are matched case-insensitively against the model path. # Order matters: more specific patterns must come before general ones. LMMS_MODEL_ADAPTERS: list[tuple[list[str], str]] = [ + # ── Audio / speech models (must come before generic "qwen" catch-all) ── + (["qwen2-audio", "qwen2_audio"], "qwen2_audio"), + (["qwen2.5-audio", "qwen2_5_audio"], "qwen2_5_audio"), + (["salmonn"], "salmonn"), + (["audio-flamingo", "audio_flamingo"], "audio_flamingo"), + (["ultravox"], "ultravox"), + (["phi-4-multimodal", "phi_4_multimodal", "phi4-multimodal"], "phi4_multimodal"), + (["gemini-audio", "gemini_audio"], "gemini_audio"), + (["gpt4o-audio", "gpt_4o_audio"], "gpt4o_audio"), + # ── Vision / video models ── (["qwen2.5-vl", "qwen2_5_vl", "qwen2.5vl"], "qwen2_5_vl"), (["qwen2-vl", "qwen2_vl"], "qwen2_vl"), (["llava-hf"], "llava_hf"), diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index ba602e2a..ce964540 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -22,26 +22,75 @@ task_metrics: bigbench_repeat_copy_logic_generate_until: exact_match bigbench_cs_algorithms_generate_until: exact_match # lmms-eval image benchmark metrics - vqav2_val_all: vqa_score + vqav2_val: exact_match mme: mme_cognition_score - mmbench_en_dev: acc - mmmu_val: acc - chartqa: relaxed_accuracy + mmbench_en_dev: gpt_eval_score + mmmu_val: mmmu_acc + chartqa: relaxed_overall docvqa_val: anls - textvqa_val: acc - ocrbench: score - # MathVista uses LLM-as-judge; null without judge LLM configured - mathvista_testmini: llm_as_judge_eval + textvqa_val: exact_match + ocrbench: ocrbench_accuracy + # MathVista uses LLM-as-judge; null without judge LLM configured. The + # top-level `mathvista_testmini` is a GROUP that expands to the three + # leaves below — schedule the leaves, not the group. mathvista_testmini_cot: llm_as_judge_eval mathvista_testmini_format: llm_as_judge_eval mathvista_testmini_solution: llm_as_judge_eval - # lmms-eval video benchmark metrics - video_mmmu: mmmu_acc + # lmms-eval video benchmark metrics. `video_mmmu` upstream is a GROUP + # that expands to perception / comprehension / adaptation leaves. + video_mmmu_perception: mmmu_acc + video_mmmu_comprehension: mmmu_acc + video_mmmu_adaptation: mmmu_acc egoschema: submission videomme: videomme_perception_score # ActivityNet-QA requires GPT API access for evaluation (LLM-as-judge) activitynetqa: gpt_eval_accuracy longvideobench_val_v: lvb_acc + librispeech_dev_clean: wer + librispeech_dev_other: wer + librispeech_test_clean: wer + librispeech_test_other: wer + common_voice_15_en: wer + common_voice_15_fr: wer + common_voice_15_zh-CN: wer + gigaspeech_test: wer + tedlium_dev_test: wer + people_speech_val: wer + voxpopuli_en: wer + ami_test: wer + wenet_speech_test_meeting: mer + wenet_speech_test_net: mer + covost2_en_zh_test: bleu + fleurs_en: wer + fleurs_cmn_hans_cn: wer + fleurs_yue_hant_hk: wer + alpaca_audio: gpt_eval + clotho_aqa_test: exact_match + openhermes: gpt_eval + wavcaps: gpt_eval + air_bench_chat_sound: gpt_eval + air_bench_chat_music: gpt_eval + air_bench_chat_speech: gpt_eval + air_bench_chat_mixed: gpt_eval + air_bench_foundation_sound: accuracy + air_bench_foundation_music: accuracy + air_bench_foundation_speech: accuracy + muchomusic: accuracy + vocalsound_test: accuracy + step2_audio_paralinguistic_age: semantic_match + step2_audio_paralinguistic_emotions: semantic_match + step2_audio_paralinguistic_event: semantic_match + step2_audio_paralinguistic_gender: semantic_match + step2_audio_paralinguistic_pitch: semantic_match + step2_audio_paralinguistic_rhythm: semantic_match + step2_audio_paralinguistic_scene: semantic_match + step2_audio_paralinguistic_speed: semantic_match + step2_audio_paralinguistic_vocalsound: semantic_match + step2_audio_paralinguistic_voice_styles: semantic_match + step2_audio_paralinguistic_voice_tone: semantic_match + cn_college_listen_mcq_test: accuracy + dream_tts_mcq_test: accuracy + voicebench_commoneval: llm_as_judge_eval task_groups: open-sci-0.01: @@ -333,8 +382,8 @@ task_groups: suite: lmms_eval n_shots: [0] tasks: - - task: vqav2_val_all - dataset: HuggingFaceM4/VQAv2 + - task: vqav2_val + dataset: lmms-lab/VQAv2 - task: mmbench_en_dev dataset: lmms-lab/MMBench - task: mmmu_val @@ -342,12 +391,17 @@ task_groups: - task: chartqa dataset: lmms-lab/ChartQA - task: docvqa_val - dataset: eliolio/docvqa + dataset: lmms-lab/DocVQA + subset: DocVQA - task: textvqa_val dataset: facebook/textvqa - task: ocrbench dataset: echo840/OCRBench - - task: mathvista_testmini + - task: mathvista_testmini_cot + dataset: AI4Math/MathVista + - task: mathvista_testmini_format + dataset: AI4Math/MathVista + - task: mathvista_testmini_solution dataset: AI4Math/MathVista # ── Individual Image Benchmarks (single-task groups for targeted runs) ────── @@ -356,8 +410,8 @@ task_groups: suite: lmms_eval n_shots: [0] tasks: - - task: vqav2_val_all - dataset: HuggingFaceM4/VQAv2 + - task: vqav2_val + dataset: lmms-lab/VQAv2 image-mmbench: description: "MMBench multi-modal benchmark via lmms-eval" @@ -389,7 +443,8 @@ task_groups: n_shots: [0] tasks: - task: docvqa_val - dataset: eliolio/docvqa + dataset: lmms-lab/DocVQA + subset: DocVQA image-textvqa: description: "TextVQA text-based visual question answering via lmms-eval" @@ -416,12 +471,14 @@ task_groups: dataset: lmms-lab/MME image-mathvista: - description: "MathVista mathematical reasoning in visual contexts via lmms-eval" + description: "MathVista mathematical reasoning in visual contexts via lmms-eval (testmini CoT/format/solution leaves)" suite: lmms_eval n_shots: [0] + dataset: AI4Math/MathVista tasks: - - task: mathvista_testmini - dataset: AI4Math/MathVista + - task: mathvista_testmini_cot + - task: mathvista_testmini_format + - task: mathvista_testmini_solution # ── Video Modality (lmms-eval) ──────────────────────────────────────────── video-understanding: @@ -429,7 +486,11 @@ task_groups: suite: lmms_eval n_shots: [0] tasks: - - task: video_mmmu + - task: video_mmmu_perception + dataset: lmms-lab/VideoMMMU + - task: video_mmmu_comprehension + dataset: lmms-lab/VideoMMMU + - task: video_mmmu_adaptation dataset: lmms-lab/VideoMMMU - task: egoschema dataset: lmms-lab/egoschema @@ -442,12 +503,14 @@ task_groups: # ── Individual Video Benchmarks (single-task groups for targeted runs) ──── video-videommmu: - description: "VideoMMMU multi-discipline video understanding via lmms-eval" + description: "VideoMMMU multi-discipline video understanding via lmms-eval (perception / comprehension / adaptation leaves)" suite: lmms_eval n_shots: [0] + dataset: lmms-lab/VideoMMMU tasks: - - task: video_mmmu - dataset: lmms-lab/VideoMMMU + - task: video_mmmu_perception + - task: video_mmmu_comprehension + - task: video_mmmu_adaptation video-egoschema: description: "EgoSchema long-form egocentric video QA via lmms-eval" @@ -481,6 +544,239 @@ task_groups: - task: longvideobench_val_v dataset: longvideobench/LongVideoBench + # Every task below MUST be a leaf (metric-emitting) task. Group names like + # air_bench_chat / common_voice_15 / fleurs expand to subtask keys at runtime, + # breaking the jobs.csv ↔ results.json match in _resolve_metric. Judge-model + # benchmarks live in individual audio-* groups and need OPENAI_API_KEY. + audio-understanding: + description: "Curated audio benchmark suite via lmms-eval — ASR (LibriSpeech, FLEURS en, GigaSpeech, TED-LIUM, WenetSpeech meeting), speech translation (CoVoST2 en→zh), audio classification (VocalSound) and music understanding (MuChoMusic). No judge-model dependency." + suite: lmms_eval + n_shots: [0] + tasks: + - task: librispeech_test_clean + dataset: lmms-lab/librispeech + - task: fleurs_en + dataset: lmms-lab/fleurs + - task: gigaspeech_test + dataset: lmms-lab/gigaspeech + - task: tedlium_dev_test + dataset: lmms-lab/tedlium + - task: wenet_speech_test_meeting + dataset: lmms-lab/WenetSpeech + - task: covost2_en_zh_test + dataset: lmms-lab/covost2_en-zh + - task: vocalsound_test + dataset: lmms-lab/vocalsound + - task: muchomusic + dataset: lmms-lab/muchomusic + + audio-librispeech: + description: "LibriSpeech ASR benchmark via lmms-eval (WER on test-clean)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: librispeech_test_clean + dataset: lmms-lab/librispeech + + audio-librispeech-all: + description: "LibriSpeech all four splits (dev-clean, dev-other, test-clean, test-other) via lmms-eval" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/librispeech + tasks: + - task: librispeech_dev_clean + - task: librispeech_dev_other + - task: librispeech_test_clean + - task: librispeech_test_other + + audio-common-voice-15: + description: "Common Voice 15 multilingual ASR via lmms-eval (en, fr, zh-CN leaf subtasks)" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/common_voice_15 + tasks: + - task: common_voice_15_en + - task: common_voice_15_fr + - task: common_voice_15_zh-CN + + audio-gigaspeech: + description: "GigaSpeech large-scale ASR via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: gigaspeech_test + dataset: lmms-lab/gigaspeech + + audio-tedlium: + description: "TED-LIUM v3 ASR via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: tedlium_dev_test + dataset: lmms-lab/tedlium + + audio-people-speech: + description: "People's Speech ASR via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: people_speech_val + dataset: lmms-lab/peoples_speech + + audio-voxpopuli: + description: "VoxPopuli multilingual ASR via lmms-eval (English split)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: voxpopuli_en + dataset: facebook/voxpopuli + + audio-ami: + description: "AMI meeting-corpus ASR via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: ami_test + dataset: edinburghcstr/ami + + audio-wenet-speech: + description: "WenetSpeech Chinese ASR via lmms-eval (MER on test_meeting / test_net)" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/WenetSpeech + tasks: + - task: wenet_speech_test_meeting + - task: wenet_speech_test_net + + audio-covost2: + description: "CoVoST2 English-Chinese speech translation via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: covost2_en_zh_test + dataset: lmms-lab/covost2_en-zh + + audio-fleurs: + description: "FLEURS multilingual speech benchmark via lmms-eval" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/fleurs + tasks: + - task: fleurs_en + - task: fleurs_cmn_hans_cn + - task: fleurs_yue_hant_hk + + audio-alpaca-audio: + description: "Alpaca-Audio spoken instruction following via lmms-eval (uses GPT-4 judge)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: alpaca_audio + dataset: lmms-lab/alpaca_audio + + audio-clotho-aqa: + description: "Clotho-AQA audio question answering via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: clotho_aqa_test + dataset: lmms-lab/ClothoAQA + + audio-openhermes: + description: "OpenHermes-Audio instruction following via lmms-eval (uses GPT-4 judge)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: openhermes + dataset: lmms-lab/openhermes_instruction + + audio-wavcaps: + description: "WavCaps audio captioning via lmms-eval (uses GPT-4 judge)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: wavcaps + dataset: AudioLLMs/wavcaps_test + + audio-air-bench-chat: + description: "AIR-Bench chat: all four subsets (sound, music, speech, mixed) via lmms-eval — uses GPT-4 judge" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/AIR_Bench + tasks: + - task: air_bench_chat_sound + - task: air_bench_chat_music + - task: air_bench_chat_speech + - task: air_bench_chat_mixed + + audio-air-bench-foundation: + description: "AIR-Bench foundation MCQ (sound, music, speech) via lmms-eval" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/AIR_Bench + tasks: + - task: air_bench_foundation_sound + - task: air_bench_foundation_music + - task: air_bench_foundation_speech + + audio-muchomusic: + description: "MuChoMusic music understanding MCQ via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: muchomusic + dataset: lmms-lab/muchomusic + + audio-vocalsound: + description: "VocalSound non-speech vocalisation classification via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: vocalsound_test + dataset: lmms-lab/vocalsound + + audio-step2-paralinguistic: + description: "Step2 audio paralinguistic probes (age, emotion, event, gender, pitch, rhythm, scene, speed, vocalsound, voice-styles, voice-tone) via lmms-eval" + suite: lmms_eval + n_shots: [0] + dataset: lmms-lab/StepEval-Audio-Paralinguistic + tasks: + - task: step2_audio_paralinguistic_age + - task: step2_audio_paralinguistic_emotions + - task: step2_audio_paralinguistic_event + - task: step2_audio_paralinguistic_gender + - task: step2_audio_paralinguistic_pitch + - task: step2_audio_paralinguistic_rhythm + - task: step2_audio_paralinguistic_scene + - task: step2_audio_paralinguistic_speed + - task: step2_audio_paralinguistic_vocalsound + - task: step2_audio_paralinguistic_voice_styles + - task: step2_audio_paralinguistic_voice_tone + + audio-cn-college-listen-mcq: + description: "Chinese college listening comprehension MCQ via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: cn_college_listen_mcq_test + dataset: AudioLLMs/cn_college_listen_mcq_test + + audio-dream-tts-mcq: + description: "DREAM TTS MCQ listening comprehension via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: dream_tts_mcq_test + dataset: AudioLLMs/dream_tts_mcq_test + + audio-voicebench: + description: "VoiceBench voice-assistant benchmark (commoneval subset) via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: voicebench_commoneval + dataset: lmms-lab/voicebench + dclm-core-22: description: "DCLM core 22 evaluation tasks (lm-eval-harness, matching LLM Foundry task types)" suite: lm-eval-harness diff --git a/oellm/task_groups.py b/oellm/task_groups.py index e925273d..eba49e6a 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -9,7 +9,7 @@ class DatasetSpec: repo_id: str subset: str | None = None - video: bool = False + needs_snapshot_download: bool = False @dataclass @@ -204,23 +204,29 @@ def _collect_dataset_specs(group_names: Iterable[str]) -> list[DatasetSpec]: def add_spec( dataset: str | None, subset: str | None, - video: bool = False, + needs_snapshot_download: bool = False, ): if dataset is None: return key = (dataset, subset) if key not in seen: seen.add(key) - specs.append(DatasetSpec(repo_id=dataset, subset=subset, video=video)) + specs.append( + DatasetSpec( + repo_id=dataset, + subset=subset, + needs_snapshot_download=needs_snapshot_download, + ) + ) for t, _, group_name in _iter_all_tasks(parsed): - is_video = group_name.startswith("video-") + needs_snapshot = group_name.startswith(("audio-", "video-")) if t.dataset == "facebook/flores" and not t.subset: for lang in _extract_flores_subsets(t.name): add_spec(t.dataset, lang) else: - add_spec(t.dataset, t.subset, video=is_video) + add_spec(t.dataset, t.subset, needs_snapshot_download=needs_snapshot) return specs diff --git a/oellm/utils.py b/oellm/utils.py index 0c381dad..976ce5a0 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -399,14 +399,16 @@ def _pre_download_datasets_from_specs( label = f"{spec.repo_id}" + (f"/{spec.subset}" if spec.subset else "") status.update(f"Downloading '{label}' ({idx}/{len(specs_list)})") - # Video datasets: lmms-eval calls snapshot_download at runtime - # to get raw video files, then symlinks them into $HF_HOME. - # Pre-download so offline compute nodes find everything cached. - if spec.video: + if spec.needs_snapshot_download: try: + # max_workers=2 keeps concurrent HEAD requests below HF's + # per-IP rate limit for many-file audio/video repos (e.g. + # lmms-lab/WenetSpeech). Higher values trigger HTTP 429 + # and long exponential backoffs even with auth. snapshot_download( repo_id=spec.repo_id, repo_type="dataset", + max_workers=2, ) except Exception as e: logging.warning(f"Failed to snapshot_download '{spec.repo_id}': {e}") diff --git a/pyproject.toml b/pyproject.toml index 37a2f8f3..e8662646 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,13 @@ image = [ video = [ "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", ] +# HPC Singularity image must also include ffmpeg for non-WAV decode. +audio = [ + "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", + "soundfile", + "librosa", + "jiwer", +] [project.scripts] oellm = "oellm.main:main" diff --git a/requirements-venv.txt b/requirements-venv.txt index 5a6c44a5..433e3ed4 100644 --- a/requirements-venv.txt +++ b/requirements-venv.txt @@ -10,6 +10,11 @@ datasets<4.0.0 # lmms-eval is compatible with datasets<4.0.0; install alongside lm-eval. lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git +# HPC Singularity image must also include ffmpeg for non-WAV decode. +soundfile +librosa +jiwer + # lighteval must be installed separately as a uv tool to avoid datasets version conflict: # UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ # uv tool install --python 3.12 --with "langcodes[data]" --with "pillow" \ diff --git a/tests/test_audio_task_groups.py b/tests/test_audio_task_groups.py new file mode 100644 index 00000000..fe048dc0 --- /dev/null +++ b/tests/test_audio_task_groups.py @@ -0,0 +1,326 @@ +import os +import sys +from importlib.resources import files +from pathlib import Path +from unittest.mock import patch + +import yaml + +from oellm.task_groups import ( + _collect_dataset_specs, + _expand_task_groups, + get_all_task_group_names, +) + +AUDIO_TASK_GROUP = "audio-understanding" + +# Tasks in the curated audio-understanding suite. All must be *leaf* tasks in +# lmms-eval (no group names that expand to subtasks at runtime) and must use +# deterministic metrics (no LLM-as-judge) so the suite runs end-to-end on a +# compute node without OPENAI_API_KEY. Judge-model and group-expanding +# benchmarks are exposed via individual audio-* groups further down. +EXPECTED_TASKS = { + "librispeech_test_clean", + "fleurs_en", + "gigaspeech_test", + "tedlium_dev_test", + "wenet_speech_test_meeting", + "covost2_en_zh_test", + "vocalsound_test", + "muchomusic", +} + +EXPECTED_DATASETS = { + "lmms-lab/librispeech", + "lmms-lab/fleurs", + "lmms-lab/gigaspeech", + "lmms-lab/tedlium", + "lmms-lab/WenetSpeech", + "lmms-lab/covost2_en-zh", + "lmms-lab/vocalsound", + "lmms-lab/muchomusic", +} + +# All individual audio-* groups that must be registered so users can target +# a single benchmark from the CLI. +INDIVIDUAL_AUDIO_GROUPS = [ + "audio-librispeech", + "audio-librispeech-all", + "audio-common-voice-15", + "audio-gigaspeech", + "audio-tedlium", + "audio-people-speech", + "audio-voxpopuli", + "audio-ami", + "audio-wenet-speech", + "audio-covost2", + "audio-fleurs", + "audio-alpaca-audio", + "audio-clotho-aqa", + "audio-openhermes", + "audio-wavcaps", + "audio-air-bench-chat", + "audio-air-bench-foundation", + "audio-muchomusic", + "audio-vocalsound", + "audio-step2-paralinguistic", + "audio-cn-college-listen-mcq", + "audio-dream-tts-mcq", + "audio-voicebench", +] + + +class TestAudioTaskGroupInRegistry: + def test_audio_understanding_present_in_yaml(self): + all_groups = get_all_task_group_names() + assert AUDIO_TASK_GROUP in all_groups + + def test_audio_understanding_suite_is_lmms_eval(self): + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + suite = data["task_groups"][AUDIO_TASK_GROUP]["suite"] + assert suite == "lmms_eval" + + def test_audio_understanding_has_eight_tasks(self): + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + tasks = data["task_groups"][AUDIO_TASK_GROUP]["tasks"] + assert len(tasks) == 8 + + def test_individual_audio_groups_present(self): + all_groups = get_all_task_group_names() + for name in INDIVIDUAL_AUDIO_GROUPS: + assert name in all_groups, f"{name} not in task group registry" + + +class TestAudioTaskGroupExpansion: + def test_expands_to_correct_task_names(self): + results = _expand_task_groups([AUDIO_TASK_GROUP]) + task_names = {r.task for r in results} + assert task_names == EXPECTED_TASKS + + def test_all_tasks_have_zero_shot(self): + results = _expand_task_groups([AUDIO_TASK_GROUP]) + for r in results: + assert r.n_shot == 0, f"{r.task} has n_shot={r.n_shot}, expected 0" + + def test_all_tasks_route_to_lmms_eval(self): + results = _expand_task_groups([AUDIO_TASK_GROUP]) + for r in results: + assert r.suite == "lmms_eval", ( + f"{r.task} has suite='{r.suite}', expected 'lmms_eval'" + ) + + def test_expand_individual_audio_group(self): + results = _expand_task_groups(["audio-librispeech"]) + assert len(results) == 1 + assert results[0].task == "librispeech_test_clean" + assert results[0].suite == "lmms_eval" + + def test_expand_librispeech_all_has_four_splits(self): + results = _expand_task_groups(["audio-librispeech-all"]) + task_names = {r.task for r in results} + assert task_names == { + "librispeech_dev_clean", + "librispeech_dev_other", + "librispeech_test_clean", + "librispeech_test_other", + } + + def test_expand_air_bench_chat_has_four_subsets(self): + results = _expand_task_groups(["audio-air-bench-chat"]) + task_names = {r.task for r in results} + assert task_names == { + "air_bench_chat_sound", + "air_bench_chat_music", + "air_bench_chat_speech", + "air_bench_chat_mixed", + } + + def test_expand_common_voice_15_has_three_languages(self): + """common_voice_15 is a group task in lmms-eval; we must list the + three language leaves explicitly so jobs.csv rows match the leaf + metric keys emitted in the output JSON.""" + results = _expand_task_groups(["audio-common-voice-15"]) + task_names = {r.task for r in results} + assert task_names == { + "common_voice_15_en", + "common_voice_15_fr", + "common_voice_15_zh-CN", + } + + def test_audio_understanding_uses_only_leaf_tasks(self): + """Regression guard: audio-understanding must not list any lmms-eval + group task (air_bench_chat, common_voice_15, fleurs, air_bench_foundation, …) + because those expand to subtask metric keys at runtime, breaking the + jobs.csv ↔ results.json match in _resolve_metric.""" + known_group_task_names = { + "air_bench_chat", + "air_bench_foundation", + "common_voice_15", + "fleurs", + "wenet_speech", + "tedlium", + "gigaspeech", + } + results = _expand_task_groups([AUDIO_TASK_GROUP]) + task_names = {r.task for r in results} + collisions = task_names & known_group_task_names + assert not collisions, ( + f"audio-understanding contains lmms-eval group tasks: {collisions}. " + "Replace with leaf subtasks (e.g. fleurs_en, wenet_speech_test_meeting)." + ) + + def test_audio_understanding_has_no_judge_tasks(self): + """Regression guard: curated smoke suite must stay runnable without + OPENAI_API_KEY on the compute node. Any task listed in task_metrics + with a gpt_eval* metric is a judge-model task and belongs in an + individual audio-* group, not the curated suite.""" + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + task_metrics = data.get("task_metrics", {}) + results = _expand_task_groups([AUDIO_TASK_GROUP]) + judge_tasks = [ + r.task + for r in results + if str(task_metrics.get(r.task, "")).startswith("gpt_eval") + ] + assert not judge_tasks, ( + f"audio-understanding contains judge-model tasks: {judge_tasks}. " + "Move them to an individual audio-* group." + ) + + +class TestAudioTaskGroupDatasetSpecs: + def test_all_expected_datasets_present(self): + specs = _collect_dataset_specs([AUDIO_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert repo_ids == EXPECTED_DATASETS + + def test_no_duplicate_dataset_specs(self): + specs = _collect_dataset_specs([AUDIO_TASK_GROUP]) + keys = [(s.repo_id, s.subset) for s in specs] + assert len(keys) == len(set(keys)), "Duplicate dataset specs found" + + def test_needs_snapshot_download_flag_set_on_specs(self): + """audio-* groups must mark their DatasetSpecs as + needs_snapshot_download=True so _pre_download_datasets_from_specs + mirrors the whole repo (load_dataset alone doesn't materialize the + loose .flac / .wav blobs that lmms-eval reads at runtime).""" + specs = _collect_dataset_specs([AUDIO_TASK_GROUP]) + assert specs, "No dataset specs returned" + for s in specs: + assert s.needs_snapshot_download, ( + f"DatasetSpec for {s.repo_id} missing needs_snapshot_download=True flag" + ) + + def test_librispeech_dataset_included(self): + specs = _collect_dataset_specs([AUDIO_TASK_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert "lmms-lab/librispeech" in repo_ids + + +class TestAudioTaskGroupScheduleEvals: + """Verify audio-understanding integrates with the schedule_evals dry-run path.""" + + def test_schedule_evals_dry_run_audio(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch( + "oellm.runner.detect_lmms_model_type", + return_value="qwen2_audio", + ), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="Qwen/Qwen2-Audio-7B-Instruct", + task_groups=AUDIO_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + sbatch_content = sbatch_files[0].read_text() + assert "lmms_eval" in sbatch_content + + def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): + import pandas as pd + + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch( + "oellm.runner.detect_lmms_model_type", + return_value="qwen2_audio", + ), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="Qwen/Qwen2-Audio-7B-Instruct", + task_groups=AUDIO_TASK_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + csv_files = list(tmp_path.glob("**/jobs.csv")) + assert len(csv_files) == 1 + df = pd.read_csv(csv_files[0]) + assert all(s.startswith("lmms_eval") for s in df["eval_suite"].unique()) + assert set(df["task_path"].unique()) == EXPECTED_TASKS + + +class TestAudioModelAdapters: + """Verify audio-specific model adapter detection.""" + + def test_qwen2_audio_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("Qwen/Qwen2-Audio-7B-Instruct") == "qwen2_audio" + + def test_qwen2_5_audio_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("Qwen/Qwen2.5-Audio-7B") == "qwen2_5_audio" + + def test_salmonn_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("tsinghua-ee/SALMONN-7B") == "salmonn" + + def test_audio_flamingo_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("nvidia/audio-flamingo-2") == "audio_flamingo" + + def test_ultravox_detected(self): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("fixie-ai/ultravox-v0_4") == "ultravox" + + def test_phi4_multimodal_detected(self): + from oellm.constants import detect_lmms_model_type + + assert ( + detect_lmms_model_type("microsoft/Phi-4-multimodal-instruct") + == "phi4_multimodal" + ) + + def test_qwen2_audio_does_not_route_to_qwen_vl(self): + """qwen2-audio must resolve to the audio adapter, not the generic qwen_vl + catch-all. This guards against ordering regressions in LMMS_MODEL_ADAPTERS.""" + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("Qwen/Qwen2-Audio-7B") == "qwen2_audio" + + def test_vision_adapters_still_work(self): + """Adding audio patterns must not regress existing vision adapter detection.""" + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("Qwen/Qwen2.5-VL-7B-Instruct") == "qwen2_5_vl" + assert detect_lmms_model_type("llava-hf/llava-1.5-7b-hf") == "llava_hf" + assert detect_lmms_model_type("lmms-lab/llava-onevision-7b") == "llava_onevision" diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index c7981a24..7d6640f5 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -112,8 +112,8 @@ def test_model_name_or_path_takes_priority_over_model_name(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/checkpoints/llava-1.5-7b", - "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, - "n-shot": {"vqav2_val_all": 0}, + "results": {"vqav2_val": {"vqav2_val/exact_match,none": 0.82}}, + "n-shot": {"vqav2_val": 0}, } df = run_collect(tmp_path, data) assert len(df) == 1 @@ -123,18 +123,18 @@ def test_task_scoped_vqa_score_key_resolved(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, - "n-shot": {"vqav2_val_all": 0}, + "results": {"vqav2_val": {"vqav2_val/exact_match,none": 0.82}}, + "n-shot": {"vqav2_val": 0}, } df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(0.82) - assert df.iloc[0]["task"] == "vqav2_val_all" + assert df.iloc[0]["task"] == "vqav2_val" - def test_task_scoped_acc_key_resolved(self, tmp_path): + def test_mmbench_gpt_eval_score_resolved(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"mmbench_en_dev": {"mmbench_en_dev/acc,none": 0.75}}, + "results": {"mmbench_en_dev": {"mmbench_en_dev/gpt_eval_score,none": 0.75}}, "n-shot": {"mmbench_en_dev": 0}, } df = run_collect(tmp_path, data) @@ -145,18 +145,18 @@ def test_mmmu_task_scoped_key(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"mmmu_val": {"mmmu/acc,none": 0.55}}, + "results": {"mmmu_val": {"mmmu_val/mmmu_acc,none": 0.55}}, "n-shot": {"mmmu_val": 0}, } df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(0.55) assert df.iloc[0]["task"] == "mmmu_val" - def test_chartqa_relaxed_accuracy(self, tmp_path): + def test_chartqa_relaxed_overall(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"chartqa": {"chartqa/relaxed_accuracy,none": 0.68}}, + "results": {"chartqa": {"chartqa/relaxed_overall,none": 0.68}}, "n-shot": {"chartqa": 0}, } df = run_collect(tmp_path, data) @@ -172,24 +172,28 @@ def test_docvqa_anls_metric(self, tmp_path): df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(0.91) - def test_ocrbench_score_metric(self, tmp_path): + def test_ocrbench_accuracy_metric(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"ocrbench": {"ocrbench/score,none": 512.0}}, + "results": {"ocrbench": {"ocrbench/ocrbench_accuracy,none": 512.0}}, "n-shot": {"ocrbench": 0}, } df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(512.0) - def test_mathvista_llm_judge_metric(self, tmp_path): + def test_mathvista_leaf_llm_judge_metric(self, tmp_path): + """mathvista_testmini is a group upstream; only the three leaves + (_cot / _format / _solution) emit the llm_as_judge_eval metric.""" data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", "results": { - "mathvista_testmini": {"mathvista_testmini/llm_as_judge_eval,none": 0.49} + "mathvista_testmini_cot": { + "mathvista_testmini_cot/llm_as_judge_eval,none": 0.49 + } }, - "n-shot": {"mathvista_testmini": 0}, + "n-shot": {"mathvista_testmini_cot": 0}, } df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(0.49) @@ -199,19 +203,19 @@ def test_multiple_image_tasks_in_one_file(self, tmp_path): "model_name": "llava_hf", "model_name_or_path": "/models/llava", "results": { - "vqav2_val_all": {"vqav2/vqa_score,none": 0.82}, - "mmbench_en_dev": {"mmbench_en_dev/acc,none": 0.75}, - "chartqa": {"chartqa/relaxed_accuracy,none": 0.68}, + "vqav2_val": {"vqav2_val/exact_match,none": 0.82}, + "mmbench_en_dev": {"mmbench_en_dev/gpt_eval_score,none": 0.75}, + "chartqa": {"chartqa/relaxed_overall,none": 0.68}, }, "n-shot": { - "vqav2_val_all": 0, + "vqav2_val": 0, "mmbench_en_dev": 0, "chartqa": 0, }, } df = run_collect(tmp_path, data) assert len(df) == 3 - assert set(df["task"].tolist()) == {"vqav2_val_all", "mmbench_en_dev", "chartqa"} + assert set(df["task"].tolist()) == {"vqav2_val", "mmbench_en_dev", "chartqa"} def test_lmeval_and_lmms_eval_results_aggregated(self, tmp_path): """Both lm-eval and lmms-eval JSON files can coexist in the same results dir.""" @@ -223,20 +227,20 @@ def test_lmeval_and_lmms_eval_results_aggregated(self, tmp_path): lmms_eval_data = { "model_name": "llava_hf", "model_name_or_path": "/path/to/model", - "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, - "n-shot": {"vqav2_val_all": 0}, + "results": {"vqav2_val": {"vqav2_val/exact_match,none": 0.82}}, + "n-shot": {"vqav2_val": 0}, } df = run_collect(tmp_path, lm_eval_data, lmms_eval_data) assert len(df) == 2 - assert set(df["task"].tolist()) == {"mmlu", "vqav2_val_all"} + assert set(df["task"].tolist()) == {"mmlu", "vqav2_val"} def test_empty_model_name_or_path_falls_back_to_model_name(self, tmp_path): """Empty string for model_name_or_path should fall back to model_name.""" data = { "model_name": "llava_hf", "model_name_or_path": "", - "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.80}}, - "n-shot": {"vqav2_val_all": 0}, + "results": {"vqav2_val": {"vqav2_val/exact_match,none": 0.80}}, + "n-shot": {"vqav2_val": 0}, } df = run_collect(tmp_path, data) assert df.iloc[0]["model_name"] == "llava_hf" @@ -245,13 +249,129 @@ def test_n_shot_zero_preserved(self, tmp_path): data = { "model_name": "llava_hf", "model_name_or_path": "/models/llava", - "results": {"vqav2_val_all": {"vqav2/vqa_score,none": 0.82}}, - "n-shot": {"vqav2_val_all": 0}, + "results": {"vqav2_val": {"vqav2_val/exact_match,none": 0.82}}, + "n-shot": {"vqav2_val": 0}, } df = run_collect(tmp_path, data) assert df.iloc[0]["n_shot"] == 0 +# ── lmms-eval audio tasks (Block A) ────────────────────────────────────────── + + +class TestCollectResultsLmmsEvalAudioFormat: + """Verify the lmms-eval output → CSV path for audio tasks. Mirrors the + image-task tests but pins the four metric families audio benchmarks use + (WER / CER / BLEU / accuracy). Each case asserts that _resolve_metric + strips the `task_name/` prefix that lmms-eval prepends to its metric keys + and looks up the right `task_metrics` entry from task-groups.yaml.""" + + def test_librispeech_wer_resolved(self, tmp_path): + """ASR task with WER metric — covers the curated audio-understanding suite.""" + data = { + "model_name": "qwen2_audio", + "model_name_or_path": "/models/qwen2-audio", + "results": { + "librispeech_test_clean": {"librispeech_test_clean/wer,none": 0.053} + }, + "n-shot": {"librispeech_test_clean": 0}, + } + df = run_collect(tmp_path, data) + assert len(df) == 1 + row = df.iloc[0] + assert row["task"] == "librispeech_test_clean" + assert row["performance"] == pytest.approx(0.053) + assert row["metric_name"] == "wer,none" + assert row["model_name"] == "/models/qwen2-audio" + + def test_wenet_speech_mer_resolved(self, tmp_path): + """Chinese ASR uses MER (Mixed Error Rate) per lmms-eval upstream.""" + data = { + "model_name": "qwen2_audio", + "model_name_or_path": "/models/qwen2-audio", + "results": { + "wenet_speech_test_meeting": {"wenet_speech_test_meeting/mer,none": 0.117} + }, + "n-shot": {"wenet_speech_test_meeting": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.117) + assert df.iloc[0]["metric_name"] == "mer,none" + + def test_covost2_bleu_resolved(self, tmp_path): + """Speech-translation task uses BLEU.""" + data = { + "model_name": "qwen2_audio", + "model_name_or_path": "/models/qwen2-audio", + "results": {"covost2_en_zh_test": {"covost2_en_zh_test/bleu,none": 24.8}}, + "n-shot": {"covost2_en_zh_test": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(24.8) + assert df.iloc[0]["metric_name"] == "bleu,none" + + def test_muchomusic_accuracy_resolved(self, tmp_path): + """Music-MCQ task uses plain accuracy.""" + data = { + "model_name": "qwen2_audio", + "model_name_or_path": "/models/qwen2-audio", + "results": {"muchomusic": {"muchomusic/accuracy,none": 0.41}}, + "n-shot": {"muchomusic": 0}, + } + df = run_collect(tmp_path, data) + assert df.iloc[0]["performance"] == pytest.approx(0.41) + assert df.iloc[0]["metric_name"] == "accuracy,none" + + def test_full_audio_understanding_suite_output(self, tmp_path): + """End-to-end: the 8 tasks in audio-understanding all resolve to + numeric performance values in a single collect_results call. This is + the regression guard that would have caught the original + air_bench_chat group-expansion bug (a group task emits subset-level + metric keys that don't match its task_path row in jobs.csv).""" + data = { + "model_name": "qwen2_audio", + "model_name_or_path": "/models/qwen2-audio", + "results": { + "librispeech_test_clean": {"librispeech_test_clean/wer,none": 0.053}, + "fleurs_en": {"fleurs_en/wer,none": 0.061}, + "gigaspeech_test": {"gigaspeech_test/wer,none": 0.094}, + "tedlium_dev_test": {"tedlium_dev_test/wer,none": 0.072}, + "wenet_speech_test_meeting": { + "wenet_speech_test_meeting/mer,none": 0.117 + }, + "covost2_en_zh_test": {"covost2_en_zh_test/bleu,none": 24.8}, + "vocalsound_test": {"vocalsound_test/accuracy,none": 0.81}, + "muchomusic": {"muchomusic/accuracy,none": 0.41}, + }, + "n-shot": dict.fromkeys( + [ + "librispeech_test_clean", + "fleurs_en", + "gigaspeech_test", + "tedlium_dev_test", + "wenet_speech_test_meeting", + "covost2_en_zh_test", + "vocalsound_test", + "muchomusic", + ], + 0, + ), + } + df = run_collect(tmp_path, data) + assert len(df) == 8 + assert all(df["performance"].notna()) + assert set(df["task"].tolist()) == { + "librispeech_test_clean", + "fleurs_en", + "gigaspeech_test", + "tedlium_dev_test", + "wenet_speech_test_meeting", + "covost2_en_zh_test", + "vocalsound_test", + "muchomusic", + } + + # ── Structured output (JSON + Markdown alongside CSV) ────────────────────── diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py index 911015c7..127398bf 100644 --- a/tests/test_image_task_groups.py +++ b/tests/test_image_task_groups.py @@ -16,22 +16,24 @@ IMAGE_TASK_GROUP = "image-vqa" EXPECTED_TASKS = { - "vqav2_val_all", + "vqav2_val", "mmbench_en_dev", "mmmu_val", "chartqa", "docvqa_val", "textvqa_val", "ocrbench", - "mathvista_testmini", + "mathvista_testmini_cot", + "mathvista_testmini_format", + "mathvista_testmini_solution", } EXPECTED_DATASETS = { - "HuggingFaceM4/VQAv2", + "lmms-lab/VQAv2", "lmms-lab/MMBench", "MMMU/MMMU", "lmms-lab/ChartQA", - "eliolio/docvqa", + "lmms-lab/DocVQA", "facebook/textvqa", "echo840/OCRBench", "AI4Math/MathVista", @@ -48,10 +50,11 @@ def test_image_vqa_suite_is_lmms_eval(self): suite = data["task_groups"][IMAGE_TASK_GROUP]["suite"] assert suite == "lmms_eval" - def test_image_vqa_has_eight_tasks(self): + def test_image_vqa_has_ten_tasks(self): data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) tasks = data["task_groups"][IMAGE_TASK_GROUP]["tasks"] - assert len(tasks) == 8 + # 7 single-metric leaves + 3 mathvista_testmini_* leaves + assert len(tasks) == 10 class TestImageTaskGroupExpansion: @@ -91,7 +94,7 @@ def test_no_duplicate_dataset_specs(self): def test_vqav2_dataset_included(self): specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) repo_ids = {s.repo_id for s in specs} - assert "HuggingFaceM4/VQAv2" in repo_ids + assert "lmms-lab/VQAv2" in repo_ids def test_mmmu_dataset_included(self): specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) diff --git a/tests/test_runner.py b/tests/test_runner.py index 59ceaf16..83ff2ae2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -55,7 +55,7 @@ def test_lmms_eval_detects_adapter(self): runner = EvalRunner() job = EvaluationJob( model_path="llava-hf/llava-1.5-7b-hf", - task_path="vqav2_val_all", + task_path="vqav2_val", n_shot=0, eval_suite="lmms_eval", ) @@ -161,7 +161,7 @@ def test_resolves_mixed_suites(self): ), EvaluationJob( model_path="llava-hf/llava-1.5-7b-hf", - task_path="vqav2_val_all", + task_path="vqav2_val", n_shot=0, eval_suite="lmms_eval", ), diff --git a/tests/test_video_task_groups.py b/tests/test_video_task_groups.py index f5d72741..21cf2e8c 100644 --- a/tests/test_video_task_groups.py +++ b/tests/test_video_task_groups.py @@ -15,7 +15,9 @@ VIDEO_TASK_GROUP = "video-understanding" EXPECTED_TASKS = { - "video_mmmu", + "video_mmmu_perception", + "video_mmmu_comprehension", + "video_mmmu_adaptation", "egoschema", "videomme", "activitynetqa", @@ -41,10 +43,11 @@ def test_video_understanding_suite_is_lmms_eval(self): suite = data["task_groups"][VIDEO_TASK_GROUP]["suite"] assert suite == "lmms_eval" - def test_video_understanding_has_five_tasks(self): + def test_video_understanding_has_seven_tasks(self): data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) tasks = data["task_groups"][VIDEO_TASK_GROUP]["tasks"] - assert len(tasks) == 5 + # 3 video_mmmu_* leaves + egoschema + videomme + activitynetqa + longvideobench + assert len(tasks) == 7 def test_individual_video_groups_present(self): all_groups = get_all_task_group_names() @@ -78,9 +81,14 @@ def test_all_tasks_route_to_lmms_eval(self): def test_expand_individual_video_group(self): results = _expand_task_groups(["video-videommmu"]) - assert len(results) == 1 - assert results[0].task == "video_mmmu" - assert results[0].suite == "lmms_eval" + assert len(results) == 3 + assert {r.task for r in results} == { + "video_mmmu_perception", + "video_mmmu_comprehension", + "video_mmmu_adaptation", + } + for r in results: + assert r.suite == "lmms_eval" class TestVideoTaskGroupDatasetSpecs: @@ -99,6 +107,18 @@ def test_videomme_dataset_included(self): repo_ids = {s.repo_id for s in specs} assert "lmms-lab/Video-MME" in repo_ids + def test_needs_snapshot_download_flag_set_on_specs(self): + """video-* groups must mark their DatasetSpecs as + needs_snapshot_download=True so _pre_download_datasets_from_specs + mirrors the whole repo (HF repos of loose .mp4 files don't round-trip + through load_dataset alone).""" + specs = _collect_dataset_specs([VIDEO_TASK_GROUP]) + assert specs, "No dataset specs returned" + for s in specs: + assert s.needs_snapshot_download, ( + f"DatasetSpec for {s.repo_id} missing needs_snapshot_download=True flag" + ) + class TestVideoTaskGroupScheduleEvals: """Verify video-understanding integrates with the schedule_evals dry-run path.""" From fdfdc4962e2773d61c2d109ef93dde2a07dbb760 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Wed, 22 Apr 2026 16:49:12 +0200 Subject: [PATCH 21/44] clean up --- oellm/scheduler.py | 8 +++- oellm/task_groups.py | 24 +++++----- tests/test_task_suite_map.py | 88 ++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 13 deletions(-) create mode 100644 tests/test_task_suite_map.py diff --git a/oellm/scheduler.py b/oellm/scheduler.py index 979fb712..af203c91 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -14,6 +14,7 @@ from oellm.constants import EvaluationJob from oellm.runner import EvalRunner from oellm.task_groups import ( + _build_task_suite_map, _collect_dataset_specs, _collect_hf_dataset_files, _collect_hf_model_repos, @@ -233,13 +234,18 @@ def schedule_evals( elif models: if group_names is None: + # Look up each bare task name in the registered groups so + # ``--tasks belebele_eng_Latn_cf`` (lighteval) or ``--tasks + # regiondial_refcocog_all`` (contrib) get routed correctly. + # Tasks not in any group default to lm_eval. + task_suite_map = _build_task_suite_map() eval_jobs.extend( [ EvaluationJob( model_path=model, task_path=task, n_shot=shot, - eval_suite="lm_eval", + eval_suite=task_suite_map.get(task, "lm_eval"), ) for model in models for task in tasks diff --git a/oellm/task_groups.py b/oellm/task_groups.py index 5793a195..d930532d 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -324,20 +324,20 @@ def _lookup_dataset_specs_for_tasks(task_names: Iterable[str]) -> list[DatasetSp def _build_task_suite_map() -> dict[str, str]: - """Build a mapping from task names to their suite from all task groups.""" - data = ( - yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) or {} - ) + """Return ``{task_name: eval_suite}`` across core YAML and contrib plugins. - task_suite_map: dict[str, str] = {} - for _, group_data in data.get("task_groups", {}).items(): - group_suite = group_data.get("suite", "lm-eval-harness") - for task_data in group_data.get("tasks", []): - task_name = task_data.get("task") - task_suite = task_data.get("suite", group_suite) - if task_name and task_name not in task_suite_map: - task_suite_map[task_name] = task_suite + Uses :func:`_parse_task_groups` + :func:`_iter_all_tasks` so contrib + registries (e.g. ``regiondial_bench``) are included, not just the core + ``task-groups.yaml``. Task-level ``suite`` overrides group-level. First + occurrence wins when a task name appears in multiple groups. + Consumers should still ``.get(task, "lm_eval")`` — tasks not registered + in any group simply aren't in the map. + """ + parsed = _parse_task_groups(get_all_task_group_names()) + task_suite_map: dict[str, str] = {} + for t, suite, _group in _iter_all_tasks(parsed): + task_suite_map.setdefault(t.name, suite) return task_suite_map diff --git a/tests/test_task_suite_map.py b/tests/test_task_suite_map.py new file mode 100644 index 00000000..c8c6be16 --- /dev/null +++ b/tests/test_task_suite_map.py @@ -0,0 +1,88 @@ +"""Tests for :func:`oellm.task_groups._build_task_suite_map`. + +The helper powers the ``--tasks`` (bare-task-name) path in the scheduler. +It must cover every suite we actually support — core YAML-registered suites +(lm-eval-harness, lighteval, lmms_eval, evalchemy) AND contrib-registered +suites (e.g. regiondial_bench). +""" + +from __future__ import annotations + +from oellm.task_groups import _build_task_suite_map + + +def test_map_is_non_empty(): + m = _build_task_suite_map() + assert len(m) > 0, "suite map must contain at least core YAML tasks" + + +def test_map_includes_lm_eval_harness_task(): + m = _build_task_suite_map() + # copa is a classic lm-eval-harness task in task-groups.yaml + assert m.get("copa") == "lm-eval-harness" + + +def test_map_includes_lighteval_task(): + m = _build_task_suite_map() + # belebele_*_cf tasks are lighteval + assert m.get("belebele_eng_Latn_cf") == "lighteval" + + +def test_map_includes_lmms_eval_task(): + """lmms_eval tasks come from image/video task groups — must be routable.""" + m = _build_task_suite_map() + # vqav2_val is the base VQA v2 task (image modality) + assert m.get("vqav2_val") == "lmms_eval" + + +def test_map_includes_contrib_task(): + """Contrib plugins (e.g. regiondial_bench) register their own TASK_GROUPS. + + These are the regression target: the original upstream helper only read + YAML and missed contrib entirely. + """ + m = _build_task_suite_map() + assert m.get("regiondial_refcocog") == "regiondial_bench" + + +def test_map_honours_task_level_suite_override(): + """Evalchemy tasks set ``suite: evalchemy`` at the task level, not the + group level — the helper must prefer the task-level value. + """ + m = _build_task_suite_map() + assert m.get("GPQADiamond") == "evalchemy" + + +def test_map_covers_all_actually_registered_suites(): + """Sanity: every distinct suite we see should be one we actually route. + + Guards against a new suite slipping into YAML or contrib without us + adding a case branch in template.sbatch (the ``*)`` catch-all routes + everything unknown to the contrib dispatcher, but we still want this + assertion as documentation). + """ + m = _build_task_suite_map() + distinct_suites = set(m.values()) + expected_subset = { + "lm-eval-harness", + "lighteval", + "lmms_eval", + "evalchemy", + "regiondial_bench", + } + # All expected suites must be present. Extra contrib suites are fine. + assert expected_subset.issubset(distinct_suites), ( + f"missing suites: {expected_subset - distinct_suites}" + ) + + +def test_first_occurrence_wins_when_task_in_multiple_groups(): + """If a task name appears in multiple groups, first occurrence wins. + + This is documented behavior of ``setdefault`` in the helper. We don't + assert a specific pair here because the YAML contents shift; we only + assert the determinism property. + """ + m1 = _build_task_suite_map() + m2 = _build_task_suite_map() + assert m1 == m2 From 35dc9061e54c3636be679ce4d637426099e91feb Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 4 May 2026 23:20:00 +0200 Subject: [PATCH 22/44] [Contrib][Audio] Add AudioBench contrib plugin (Part 1: judge-free tasks) * Add AudioBench contrib plugin (Part 1: judge-free tasks) * Refactor ready and logic for venv management --- docs/VENV.md | 24 +- oellm/contrib/audiobench/README.md | 179 +++++ oellm/contrib/audiobench/__init__.py | 0 oellm/contrib/audiobench/adapter.py | 69 ++ oellm/contrib/audiobench/suite.py | 320 ++++++++ oellm/contrib/audiobench/task.py | 192 +++++ oellm/contrib/regiondial_bench/README.md | 57 +- oellm/resources/template.sbatch | 5 +- oellm/scheduler.py | 11 +- pyproject.toml | 13 + tests/test_audiobench.py | 946 +++++++++++++++++++++++ 11 files changed, 1785 insertions(+), 31 deletions(-) create mode 100644 oellm/contrib/audiobench/README.md create mode 100644 oellm/contrib/audiobench/__init__.py create mode 100644 oellm/contrib/audiobench/adapter.py create mode 100644 oellm/contrib/audiobench/suite.py create mode 100644 oellm/contrib/audiobench/task.py create mode 100644 tests/test_audiobench.py diff --git a/docs/VENV.md b/docs/VENV.md index 553500fc..da9f53bb 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -4,7 +4,29 @@ Instead of using pre-built containers, you can run evaluations with your own Python virtual environment by passing `--venv-path`. -## Setup +## Choosing your venv + +Most evaluations share **one general venv**. A handful of framework-level +suites have hard dependency conflicts and need their own venv: + +| Task group(s) | Engine | Venv | Setup | +|---|---|---|---| +| `open-sci-*`, `belebele_*_cf`, all text/multilingual tasks | `lm-eval-harness`, `lighteval` | **general** | [Setup](#setup-general-venv) | +| `image-*`, `video-*`, `audio-*` (modality-prefixed) | `lmms-eval` | **general** | [Setup](#setup-general-venv) | +| `dclm-core-22` | `lm-eval-harness` (pinned 0.4.9.2) | **dclm** | [DCLM-core-22](#dclm-core-22) | +| `reasoning` (GPQA/MATH500/AIME/MBPP/etc.) | `evalchemy` + forked lm-eval | **evalchemy** | [Evalchemy](#evalchemy-reasoning) | + +Custom contrib benchmarks bring their own dependency stacks and are +documented in `oellm/contrib//README.md`: + +| Task group(s) | Contrib | README | +|---|---|---| +| `audio-audiobench*` | `audiobench` | [`oellm/contrib/audiobench/README.md`](../oellm/contrib/audiobench/README.md) | +| `regiondial-*` | `regiondial_bench` | [`oellm/contrib/regiondial_bench/README.md`](../oellm/contrib/regiondial_bench/README.md) | + +Use `oellm list-tasks` to see which suite a given task group routes to. + +## Setup (general venv) 1. Create a venv with Python 3.12: ```bash diff --git a/oellm/contrib/audiobench/README.md b/oellm/contrib/audiobench/README.md new file mode 100644 index 00000000..60ab3261 --- /dev/null +++ b/oellm/contrib/audiobench/README.md @@ -0,0 +1,179 @@ +# AudioBench + +AudioBench (AudioLLMs/AudioBench, [arXiv 2406.16020](https://arxiv.org/abs/2406.16020)) +is a broad audio-understanding benchmark covering ASR, speech translation, +spoken reasoning, audio scene QA, and paralinguistics. This contrib plugin +wraps AudioBench as a callable `audiobench` suite inside elliot-cli so WP4 +can produce numbers directly comparable with the AudioBench paper and +leaderboard, without the scoring-normalisation drift that would come from +running the same datasets through lmms-eval. + +## Scope + +**27 judge-free tasks** across ASR (WER), speech translation (BLEU), spoken +reasoning (accuracy / string_match), and AudioCaps (METEOR). Of these: + +- **20 tasks are genuinely new** to the platform — not in any of our + existing lmms-eval `audio-*` groups. Examples: `earnings21_test`, + `earnings22_test`, GigaSpeech2 (Thai / Indonesian / Vietnamese), + SEAME code-switch, Spoken-MQA reasoning splits, MMAU mini. +- **7 tasks are dual-registered** duplicates of benchmarks we already run + through lmms-eval (LibriSpeech test-clean/other, Common Voice 15 EN, + GigaSpeech, People's Speech, TED-LIUM 3, CoVoST2 en→zh). These use + AudioBench's own scorer and normaliser so WP4 can report numbers + aligned with the AudioBench paper. + +Every AudioBench task is namespaced with an `audiobench_` prefix so the CSV +`task_path` column unambiguously identifies which scorer produced a number +(e.g. `audiobench_librispeech_test_clean` is AudioBench-scored; +`librispeech_test_clean` remains the lmms-eval version). + +Judge-dependent tasks (SLUE-SQA5, Spoken-SQuAD, AudioCaps-QA, IEMOCAP / +MELD / VoxCeleb probes, AudioLLM-InstructionFollowing) are not included +and depend on a vLLM judge service being provisioned on Leonardo. + +## Prerequisites + +AudioBench is not pip-installable (no upstream build backend, bare imports +in `src/main_evaluate.py`); the plugin invokes it as a subprocess from an +on-cluster clone. A dedicated venv is required: the `[audiobench]` extra +pins `transformers<5` and `jiwer<3`, which conflict with the general eval +venv (see [`docs/VENV.md`](../../../docs/VENV.md) for the framework venvs). + +### 1. Clone AudioBench and configure `clusters.yaml` + +```bash +git clone https://github.com/AudioLLMs/AudioBench /path/to/AudioBench +``` + +Add `AUDIOBENCH_DIR` to your cluster block in +`oellm/resources/clusters.yaml`: + +```yaml +leonardo: + ... + AUDIOBENCH_DIR: "/path/to/AudioBench" +``` + +### 2. Create the venv + +```bash +uv venv --python 3.12 audiobench-venv +source audiobench-venv/bin/activate +uv pip install -e ".[audiobench]" +``` + +The `[audiobench]` extra pins `transformers>=4.45,<5`, `jiwer<3`, +`sacrebleu`, `pythainlp`, `evaluate`, `soundfile`, `librosa`. + +### 3. Install AudioBench's runtime dependencies + +```bash +# AudioBench's own requirements (filter vllm; only used by deferred judge tasks) +grep -v -i '^vllm' /path/to/AudioBench/requirements.txt > /tmp/ab-reqs.txt +uv pip install -r /tmp/ab-reqs.txt + +# PyTorch for cluster's CUDA driver — PyPI defaults target a newer runtime +# than most HPC drivers (Leonardo / JURECA report CUDA 12.2) and crash with +# `NVIDIA driver too old`. Use the cu121 index. +uv pip install torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/cu121 + +# rapidfuzz C extension — without this, jiwer's WER scoring hits the +# pure-Python fallback and raises NotImplementedError on Levenshtein.editops. +uv pip install --reinstall rapidfuzz +``` + +> Verify the venv works: +> ```bash +> python -c " +> from transformers import Qwen2AudioForConditionalGeneration +> from rapidfuzz.distance import Levenshtein +> Levenshtein.editops('a', 'b') # must not raise +> print('audiobench venv OK') +> " +> ``` + +### Dataset pre-download + +No manual steps required. `schedule-eval` pre-downloads every +`AudioLLMs/*` HF repo referenced by the requested task group on the login +node via `huggingface_hub.snapshot_download(max_workers=2)`, so compute +nodes do not need internet access. + +## Running + +### Available task groups + +| Task group | Leaves | What it covers | +|----------------------------------|--------|-----------------------------------------------------------------| +| `audio-audiobench` | 27 | Full suite (everything below). | +| `audio-audiobench-asr` | 15 | WER tasks — 9 new + 6 dual-registered with lmms-eval. | +| `audio-audiobench-st` | 6 | BLEU speech-translation — 5 new + 1 dual (en→zh). | +| `audio-audiobench-reasoning` | 6 | Spoken-MQA × 4, MMAU mini, AudioCaps METEOR. | + +### Example + +```bash +# Full AudioBench suite on a Qwen2-Audio model: +oellm schedule-eval \ + --models Qwen/Qwen2-Audio-7B-Instruct \ + --task-groups audio-audiobench \ + --venv-path audiobench-venv + +# ASR only: +oellm schedule-eval \ + --models Qwen/Qwen2-Audio-7B-Instruct \ + --task-groups audio-audiobench-asr \ + --venv-path audiobench-venv + +# Smoke test with --limit: +oellm schedule-eval \ + --models Qwen/Qwen2-Audio-7B-Instruct \ + --task-groups audio-audiobench-asr \ + --limit 100 \ + --venv-path audiobench-venv +``` + +`--limit N` is forwarded to AudioBench's `--number_of_samples N`. When +unset, the full test split is evaluated. + +### Collecting results + +```bash +oellm collect-results \ + --eval-output-dir /path/to/evals \ + --output-csv audiobench_results.csv +``` + +The primary metric per task is what's registered in `task_metrics` +(`wer` / `bleu` / `accuracy` / `string_match` / `meteor`). Dual-registered +tasks land in the CSV **alongside** their lmms-eval counterparts, with +different `task_path` values (`audiobench_librispeech_test_clean` vs +`librispeech_test_clean`) and different `eval_suite` values (`audiobench` +vs `lmms_eval`) — no silent averaging. + +## Supported model adapters + +AudioBench dispatches on a fixed list of literal `model_name` strings +(see `$AUDIOBENCH_DIR/src/model.py`); each loader under `model_src/` +fetches its own HF repo. Arbitrary HF checkpoints are not supported — +only the variants below: + +| Model path substring (lowered) | AudioBench `model_name` (literal) | +|------------------------------------------------|-------------------------------------------| +| `qwen2-audio-7b-instruct` / `qwen2_audio_7b_instruct` | `Qwen2-Audio-7B-Instruct` | +| `qwen-audio-chat` / `qwen_audio_chat` | `Qwen-Audio-Chat` | +| `salmonn` | `SALMONN_7B` | +| `meralion-audiollm` / `meralion_audiollm` | `MERaLiON-AudioLLM-Whisper-SEA-LION` | +| `whisper-large-v3` / `whisper_large_v3` | `whisper_large_v3` | +| `whisper-large-v2` / `whisper_large_v2` | `whisper_large_v2` | +| `phi-4-multimodal` / `phi_4_multimodal` | `phi_4_multimodal_instruct` | +| `seallms-audio-7b` / `seallms_audio_7b` | `seallms_audio_7b` | +| `wavllm` | `WavLLM_fairseq` | +| (anything else) | error — no generic loader upstream | + +To override detection, pass the literal AudioBench key as a suffix: +`audiobench:Qwen2-Audio-7B-Instruct`. Case is preserved end-to-end +(AudioBench's match is case-sensitive). + diff --git a/oellm/contrib/audiobench/__init__.py b/oellm/contrib/audiobench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oellm/contrib/audiobench/adapter.py b/oellm/contrib/audiobench/adapter.py new file mode 100644 index 00000000..66a9655a --- /dev/null +++ b/oellm/contrib/audiobench/adapter.py @@ -0,0 +1,69 @@ +"""AudioBench model adapter. + +Maps a HuggingFace model path to AudioBench's literal ``--model_name`` value. + +AudioBench's ``Model`` class (in ``$AUDIOBENCH_DIR/src/model.py``) dispatches +on **exact-string** match against a fixed list — there is no family-level +indirection and no fallback. Each supported model has a hardcoded loader +under ``model_src/`` that loads its own HF repo internally; AudioBench +**cannot evaluate arbitrary HF checkpoints**, only the variants it knows +about. If we can't map the user's ``model_path`` to one of those literals, +we return ``None`` and ``suite.run`` raises a clear error. +""" + +from __future__ import annotations + +from oellm.core.base_model_adapter import BaseModelAdapter + +# (audiobench_model_name, substrings_to_match_in_lower(model_path)). +# Order matters — first match wins; put more-specific patterns first. +# Keys MUST be the exact literals AudioBench's model.py dispatch expects. +_PATTERNS: list[tuple[str, tuple[str, ...]]] = [ + ("Qwen2-Audio-7B-Instruct", ("qwen2-audio-7b-instruct", "qwen2_audio_7b_instruct")), + ("Qwen-Audio-Chat", ("qwen-audio-chat", "qwen_audio_chat")), + ("SALMONN_7B", ("salmonn",)), + ("MERaLiON-AudioLLM-Whisper-SEA-LION", ("meralion-audiollm", "meralion_audiollm")), + ("whisper_large_v3", ("whisper-large-v3", "whisper_large_v3")), + ("whisper_large_v2", ("whisper-large-v2", "whisper_large_v2")), + ("phi_4_multimodal_instruct", ("phi-4-multimodal", "phi_4_multimodal")), + ("seallms_audio_7b", ("seallms-audio-7b", "seallms_audio_7b")), + ("WavLLM_fairseq", ("wavllm",)), +] + + +class AudioBenchModelAdapter(BaseModelAdapter): + """Adapter resolving the ``--model_name`` value for the AudioBench subprocess.""" + + def __init__(self, model_path: str) -> None: + self._path = model_path + + @property + def model_path(self) -> str: + return self._path + + def to_lm_eval_args(self) -> str: + # Unused — AudioBench doesn't route through lm-eval. Required by + # BaseModelAdapter. + return f"pretrained={self._path},trust_remote_code=True" + + def to_lmms_eval_args(self) -> str: + # Unused — see to_lm_eval_args(). + return f"pretrained={self._path}" + + def to_contrib_flags(self) -> str | None: + """Return AudioBench's ``model_name`` dispatch key, or ``None`` if no match. + + Returning ``None`` is intentional: AudioBench has no generic loader, + so an unmatched model path must fail loudly rather than fall through + to a fictitious ``generic`` key that AudioBench doesn't recognize. + """ + lowered = self._path.lower() + for key, needles in _PATTERNS: + if any(n in lowered for n in needles): + return key + return None + + +def detect_audiobench_model_type(model_path: str) -> str | None: + """Convenience wrapper around :meth:`AudioBenchModelAdapter.to_contrib_flags`.""" + return AudioBenchModelAdapter(model_path).to_contrib_flags() diff --git a/oellm/contrib/audiobench/suite.py b/oellm/contrib/audiobench/suite.py new file mode 100644 index 00000000..7ec28e46 --- /dev/null +++ b/oellm/contrib/audiobench/suite.py @@ -0,0 +1,320 @@ +"""AudioBench contrib suite — plugin protocol implementation. + +AudioBench is not pip-installable (upstream has no build backend and uses +bare imports like ``from dataset import ...``), so :func:`run` invokes its +``src/main_evaluate.py`` entry point as a subprocess with ``cwd`` set to +``$AUDIOBENCH_DIR``. :func:`run` then re-shapes AudioBench's result JSON +into a lmms-eval-compatible payload that :func:`oellm.main.collect_results` +can parse unchanged. +""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +from pathlib import Path + +from oellm.contrib.audiobench.task import ( + AUDIOBENCH_TASKS, + SUITE_NAME, + AudioBenchTaskSpec, + get_task_spec, +) + +logger = logging.getLogger(__name__) + +CLUSTER_ENV_VARS = ["AUDIOBENCH_DIR"] + +_FAMILY_GROUPS = { + "asr": ( + "audio-audiobench-asr", + "AudioBench ASR tasks (WER).", + ), + "st": ( + "audio-audiobench-st", + "AudioBench speech-translation tasks (BLEU).", + ), + "reasoning": ( + "audio-audiobench-reasoning", + "AudioBench spoken reasoning / captioning (accuracy / string_match / METEOR).", + ), +} + +_TOP_LEVEL_GROUP = "audio-audiobench" +_TOP_LEVEL_DESC = ( + "AudioBench suite — ASR (WER), speech translation (BLEU), spoken " + "reasoning (accuracy/string_match), and AudioCaps captioning (METEOR)." +) + + +def _build_task_groups() -> dict: + """Build ``TASK_GROUPS`` from :data:`AUDIOBENCH_TASKS`. + + Always zero-shot — AudioBench does not support in-context examples. + """ + task_metrics: dict[str, str] = {t.name: t.metric for t in AUDIOBENCH_TASKS} + + def _task_entry(t: AudioBenchTaskSpec) -> dict: + # No ``subset`` — for gigaspeech2 / spoken-mqa the upstream split + # selection is encoded in ``upstream_name`` itself (e.g. + # ``gigaspeech2_thai``). The ``audio-*`` group prefix triggers + # full-repo snapshot_download in :func:`_collect_dataset_specs`. + return {"task": t.name, "dataset": t.hf_repo} + + groups: dict[str, dict] = {} + + tasks_by_family: dict[str, list[AudioBenchTaskSpec]] = { + "asr": [], + "st": [], + "reasoning": [], + } + for t in AUDIOBENCH_TASKS: + tasks_by_family[t.family].append(t) + + for family, (group_name, desc) in _FAMILY_GROUPS.items(): + entries = tasks_by_family[family] + if not entries: + continue + groups[group_name] = { + "suite": SUITE_NAME, + "n_shots": [0], + "description": desc, + "tasks": [_task_entry(t) for t in entries], + } + + groups[_TOP_LEVEL_GROUP] = { + "suite": SUITE_NAME, + "n_shots": [0], + "description": _TOP_LEVEL_DESC, + "tasks": [_task_entry(t) for t in AUDIOBENCH_TASKS], + } + + return {"task_metrics": task_metrics, "task_groups": groups} + + +TASK_GROUPS: dict = _build_task_groups() + + +def detect_model_flags(model_path: str) -> str | None: + """Return AudioBench's literal ``--model_name`` dispatch key for *model_path*. + + Returns ``None`` when *model_path* does not match any AudioBench-supported + model family — :func:`run` then raises a clear error. AudioBench has no + generic loader, so silently falling back to a fictitious key would just + move the error deeper inside the subprocess. + """ + from oellm.contrib.audiobench.adapter import AudioBenchModelAdapter + + return AudioBenchModelAdapter(model_path).to_contrib_flags() + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Execute one AudioBench task and write a lmms-eval-shaped result JSON. + + Raises ``RuntimeError`` if AudioBench exits non-zero or produces no + parseable output, and ``KeyError`` if *task* is not registered. + """ + ab_dir = env.get("AUDIOBENCH_DIR") + if not ab_dir: + raise RuntimeError( + "AUDIOBENCH_DIR must be set. Add it to clusters.yaml — " + "it should point at a local clone of " + "https://github.com/AudioLLMs/AudioBench." + ) + + entrypoint = Path(ab_dir) / "src" / "main_evaluate.py" + if not entrypoint.exists(): + raise FileNotFoundError( + f"AudioBench entry point not found: {entrypoint}\n" + f"Check that AUDIOBENCH_DIR={ab_dir!r} points at a valid " + "AudioBench clone." + ) + + spec = get_task_spec(task) + if not model_flags: + raise RuntimeError( + f"Could not map model_path={model_path!r} to an AudioBench-supported " + f"model. AudioBench dispatches on a fixed list of literal " + f"model_name strings (Qwen2-Audio-7B-Instruct, SALMONN_7B, " + f"whisper_large_v3, …) — see oellm/contrib/audiobench/adapter.py. " + f"AudioBench cannot evaluate arbitrary HF checkpoints; it loads " + f"its own hardcoded HF repos per model family." + ) + model_key = model_flags # AudioBench's dispatch key, e.g. "Qwen2-Audio-7B-Instruct" + + cmd = [ + "python", + "src/main_evaluate.py", + "--dataset_name", + spec.upstream_name, + "--model_name", + model_key, + "--metrics", + spec.upstream_metric, + # Force re-eval — AudioBench skips by default if a stale score file + # already exists under log_for_all_models/. + "--overwrite", + "True", + ] + + limit = env.get("LIMIT", "").strip() + if limit: + cmd.extend(["--number_of_samples", str(limit)]) + + logger.info("AudioBench cmd: %s (cwd=%s)", " ".join(cmd), ab_dir) + completed = subprocess.run( + cmd, + cwd=ab_dir, + env=env, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"AudioBench exited with code {completed.returncode} for " + f"task={task!r} model={model_path!r} (dispatch key={model_key!r})" + ) + + metrics = _extract_metrics( + audiobench_dir=Path(ab_dir), model_key=model_key, spec=spec + ) + _write_lmms_shaped_json( + output_path=output_path, + model_path=model_path, + task_name=task, + n_shot=n_shot, + metrics=metrics, + ) + logger.info("Results written to %s", output_path) + + +def _extract_metrics( + *, + audiobench_dir: Path, + model_key: str, + spec: AudioBenchTaskSpec, +) -> dict[str, float]: + """Read AudioBench's score file from its hardcoded output path. + + AudioBench writes to ``$cwd/log_for_all_models//__score.json`` + (see ``main_evaluate.py:118``). Path is fixed — there is no ``--log_dir``. + """ + score_file = ( + audiobench_dir + / "log_for_all_models" + / model_key + / f"{spec.upstream_name}_{spec.upstream_metric}_score.json" + ) + if not score_file.exists(): + raise RuntimeError( + f"AudioBench did not write expected score file at {score_file}. " + f"Either AudioBench crashed silently, or the dispatch key " + f"{model_key!r} / dataset_name {spec.upstream_name!r} / metric " + f"{spec.upstream_metric!r} is wrong. Check stdout/stderr." + ) + + try: + with open(score_file) as f: + body = json.load(f) + except (json.JSONDecodeError, OSError) as e: + raise RuntimeError( + f"Could not read AudioBench score file {score_file}: {e}" + ) from e + + value = _find_metric(body, spec.upstream_metric) + if value is None: + raise RuntimeError( + f"Could not locate metric {spec.upstream_metric!r} in AudioBench " + f"score file {score_file}. Body: {body!r}" + ) + # Emit under our canonical key so collect_results' metric resolution + # picks up task_metrics.yaml. + return {spec.metric: float(value)} + + +def _find_metric(body: object, key: str) -> float | None: + """Recursive search for a numeric value keyed by *key*. + + Tolerates both ``{"wer": 0.04}`` and ``{"metrics": {"wer": {"score": + 0.04}}}`` layouts — upstream log shape has drifted across releases. + """ + if isinstance(body, dict): + if key in body: + candidate = body[key] + if isinstance(candidate, int | float): + return float(candidate) + if isinstance(candidate, dict) and "score" in candidate: + score = candidate["score"] + if isinstance(score, int | float): + return float(score) + for v in body.values(): + found = _find_metric(v, key) + if found is not None: + return found + elif isinstance(body, list): + for item in body: + found = _find_metric(item, key) + if found is not None: + return found + return None + + +def _write_lmms_shaped_json( + *, + output_path: Path, + model_path: str, + task_name: str, + n_shot: int, + metrics: dict[str, float], +) -> None: + payload = { + "model_name_or_path": model_path, + "results": {task_name: metrics}, + "configs": {task_name: {"num_fewshot": n_shot}}, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(payload, f, indent=2) + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Recognise a JSON dict produced by :func:`run` and return + ``(model_id, task_name, n_shot, metrics)``; ``None`` if it's not ours. + """ + results = data.get("results", {}) + if not isinstance(results, dict): + return None + for task_name, task_results in results.items(): + if not isinstance(task_name, str) or not task_name.startswith("audiobench_"): + continue + if not isinstance(task_results, dict): + continue + model_id = data.get("model_name_or_path") or data.get("model_name") or "unknown" + n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) + coerced: dict[str, float] = {} + for k, v in task_results.items(): + if isinstance(v, int | float): + coerced[k] = float(v) + return model_id, task_name, int(n_shot), coerced + return None + + +__all__ = [ + "CLUSTER_ENV_VARS", + "SUITE_NAME", + "TASK_GROUPS", + "detect_model_flags", + "parse_results", + "run", +] + +_ = os # exported via env dict passed to subprocess.run diff --git a/oellm/contrib/audiobench/task.py b/oellm/contrib/audiobench/task.py new file mode 100644 index 00000000..32098880 --- /dev/null +++ b/oellm/contrib/audiobench/task.py @@ -0,0 +1,192 @@ +"""AudioBench task registry. + +Single source of truth for the task set. Consumed by +:mod:`oellm.contrib.audiobench.suite` to build ``TASK_GROUPS`` and to look up +per-task metadata (HF repo, upstream task name, metric) at dispatch time. + +Every canonical task name is prefixed ``audiobench_`` so the CSV ``task_path`` +column uniquely identifies the scorer and doesn't collide with lmms-eval's +names for the same benchmark. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +SUITE_NAME = "audiobench" +_TASK_NAME_PREFIX = "audiobench_" + + +@dataclass(frozen=True) +class AudioBenchTaskSpec: + """Metadata for a single AudioBench task. + + ``upstream_name`` is the literal string AudioBench's ``--dataset_name`` + expects (matched exactly against ``$AUDIOBENCH_DIR/src/dataset.py``'s + dispatch table). ``upstream_metric`` is what ``--metrics`` expects + (usually identical to our canonical ``metric``). + """ + + name: str + upstream_name: str + hf_repo: str + metric: str + upstream_metric: str + family: str + + @property + def task_group(self) -> str: + return f"audio-audiobench-{self.family}" + + +def _t( + upstream_name: str, + hf_repo: str, + metric: str, + family: str, + *, + upstream_metric: str | None = None, + name: str | None = None, +) -> AudioBenchTaskSpec: + """Build a spec with ``name = audiobench_`` by default.""" + return AudioBenchTaskSpec( + name=name if name is not None else _TASK_NAME_PREFIX + upstream_name, + upstream_name=upstream_name, + hf_repo=hf_repo, + metric=metric, + upstream_metric=upstream_metric or metric, + family=family, + ) + + +# Tasks not covered by our lmms-eval task groups. +_NEW_ASR = [ + _t("aishell_asr_zh_test", "AudioLLMs/aishell_1_zh_test", "wer", "asr"), + _t("earnings21_test", "AudioLLMs/earnings21_test", "wer", "asr"), + _t("earnings22_test", "AudioLLMs/earnings22_test", "wer", "asr"), + _t("tedlium3_long_form_test", "AudioLLMs/tedlium3_long_form_test", "wer", "asr"), + # GigaSpeech2 — 3 languages share one HF repo. AudioBench dispatches via + # the dataset_name string itself (gigaspeech2_thai/indo/viet), not via a + # --data_dir flag (which doesn't exist upstream). + _t( + "gigaspeech2_thai", + "AudioLLMs/gigaspeech2-test", + "wer", + "asr", + name="audiobench_gigaspeech2_thai", + ), + _t( + "gigaspeech2_indo", + "AudioLLMs/gigaspeech2-test", + "wer", + "asr", + name="audiobench_gigaspeech2_indo", + ), + _t( + "gigaspeech2_viet", + "AudioLLMs/gigaspeech2-test", + "wer", + "asr", + name="audiobench_gigaspeech2_viet", + ), + _t("seame_dev_man", "AudioLLMs/seame_dev_man", "wer", "asr"), + _t("seame_dev_sge", "AudioLLMs/seame_dev_sge", "wer", "asr"), +] + +_NEW_ST = [ + _t("covost2_en_id_test", "AudioLLMs/covost2_en_id_test", "bleu", "st"), + _t("covost2_en_ta_test", "AudioLLMs/covost2_en_ta_test", "bleu", "st"), + _t("covost2_id_en_test", "AudioLLMs/covost2_id_en_test", "bleu", "st"), + _t("covost2_zh_en_test", "AudioLLMs/covost2_zh_en_test", "bleu", "st"), + _t("covost2_ta_en_test", "AudioLLMs/covost2_ta_en_test", "bleu", "st"), +] + +_NEW_REASONING = [ + # Spoken-MQA — 4 splits share one HF repo. AudioBench dispatches via + # the hyphen-prefixed dataset_name (spoken-mqa_), not --data_dir. + _t( + "spoken-mqa_short_digit", + "amao0o0/spoken-mqa", + "accuracy", + "reasoning", + upstream_metric="acc", + name="audiobench_spoken_mqa_short_digit", + ), + _t( + "spoken-mqa_long_digit", + "amao0o0/spoken-mqa", + "accuracy", + "reasoning", + upstream_metric="acc", + name="audiobench_spoken_mqa_long_digit", + ), + _t( + "spoken-mqa_single_step_reasoning", + "amao0o0/spoken-mqa", + "accuracy", + "reasoning", + upstream_metric="acc", + name="audiobench_spoken_mqa_single_step_reasoning", + ), + _t( + "spoken-mqa_multi_step_reasoning", + "amao0o0/spoken-mqa", + "accuracy", + "reasoning", + upstream_metric="acc", + name="audiobench_spoken_mqa_multi_step_reasoning", + ), + _t("mmau_mini", "AudioLLMs/MMAU-mini", "string_match", "reasoning"), + _t("audiocaps_test", "AudioLLMs/audiocaps_test", "meteor", "reasoning"), +] + +# Dual-registered duplicates of benchmarks also in lmms-eval. These use +# AudioBench's scorer/normaliser for paper-comparable numbers; the lmms-eval +# versions stay in place. HF repos differ (AudioLLMs/* vs lmms-lab/*) so +# snapshot_download does not collide. +_DUAL = [ + _t("librispeech_test_clean", "AudioLLMs/librispeech_test_clean", "wer", "asr"), + _t("librispeech_test_other", "AudioLLMs/librispeech_test_other", "wer", "asr"), + _t("common_voice_15_en_test", "AudioLLMs/common_voice_15_en_test", "wer", "asr"), + _t("gigaspeech_test", "AudioLLMs/gigaspeech_test", "wer", "asr"), + _t("peoples_speech_test", "AudioLLMs/peoples_speech_test", "wer", "asr"), + _t("tedlium3_test", "AudioLLMs/tedlium3_test", "wer", "asr"), + _t("covost2_en_zh_test", "AudioLLMs/covost2_en_zh_test", "bleu", "st"), +] + + +AUDIOBENCH_TASKS: list[AudioBenchTaskSpec] = [ + *_NEW_ASR, + *_NEW_ST, + *_NEW_REASONING, + *_DUAL, +] + + +def _validate() -> None: + seen_names: set[str] = set() + for t in AUDIOBENCH_TASKS: + if t.name in seen_names: + raise RuntimeError(f"Duplicate AudioBench task name {t.name!r} in registry") + seen_names.add(t.name) + if not t.name.startswith(_TASK_NAME_PREFIX): + raise RuntimeError( + f"AudioBench task {t.name!r} missing required prefix " + f"{_TASK_NAME_PREFIX!r}" + ) + if t.family not in {"asr", "st", "reasoning"}: + raise RuntimeError( + f"AudioBench task {t.name!r} has unknown family {t.family!r}" + ) + + +_validate() + + +def get_task_spec(name: str) -> AudioBenchTaskSpec: + """Look up a spec by canonical task name; raises ``KeyError`` if missing.""" + for t in AUDIOBENCH_TASKS: + if t.name == name: + return t + known = sorted(t.name for t in AUDIOBENCH_TASKS) + raise KeyError(f"Unknown AudioBench task {name!r}. Known tasks: {', '.join(known)}") diff --git a/oellm/contrib/regiondial_bench/README.md b/oellm/contrib/regiondial_bench/README.md index b540d6ef..57f646ea 100644 --- a/oellm/contrib/regiondial_bench/README.md +++ b/oellm/contrib/regiondial_bench/README.md @@ -16,17 +16,17 @@ plus per-round breakdown (R1–R7) for gIoU and bbox_AP. ## Prerequisites -### 1. Clone RegionReasoner +The benchmark calls `test/evaluation/evaluation_multi_segmentation.py` and +the `test/vision_reasoner/` model wrapper from the RegionReasoner +repository as a subprocess, so the repo must be present on the cluster +filesystem. A dedicated venv is required for `flash-attn` (specific +pre-built wheel) and HEIF image support (`pi-heif`); see +[`docs/VENV.md`](../../../docs/VENV.md) for the framework venvs. -The benchmark relies on the inference script -`test/evaluation/evaluation_multi_segmentation.py` and the model wrapper -`test/vision_reasoner/` from the RegionReasoner repository. These are **not -packaged** — the platform calls them directly as a subprocess, so the repo -must be present on the cluster filesystem. +### 1. Clone RegionReasoner ```bash -git clone https://github.com/lmsdss/RegionReasoner \ - /path/to/RegionReasoner +git clone https://github.com/lmsdss/RegionReasoner /path/to/RegionReasoner ``` ### 2. Configure clusters.yaml @@ -38,40 +38,41 @@ my-cluster: ... HF_HOME: "/path/to/large/filesystem/huggingface" # must have ~30 GB free REGION_REASONER_DIR: "/path/to/RegionReasoner" - GPUS_PER_NODE: 4 # controls both SLURM --gres and shard count + GPUS_PER_NODE: 4 # controls SLURM --gres and shard count ``` -> **`HF_HOME`** must point to a filesystem with at least **30 GB** of free -> space. On CINECA Leonardo, use the work filesystem -> (`/leonardo_work//huggingface`), not the home filesystem (50 GB -> quota, fills up quickly). +> `HF_HOME` must point to a filesystem with at least 30 GB free. On +> CINECA Leonardo, use the work filesystem +> (`/leonardo_work//huggingface`), not the home filesystem +> (50 GB quota). -### 3. Install dependencies in your venv +### 3. Create a venv and install dependencies ```bash -# PyTorch — match the CUDA version available on your cluster -pip install torch==2.5.1 --index-url https://download.pytorch.org/whl/cu121 +uv venv --python 3.12 regiondial-venv +source regiondial-venv/bin/activate +uv pip install -e . -# Matching torchvision -pip install torchvision==0.20.1 --index-url https://download.pytorch.org/whl/cu121 +# PyTorch — match the cluster's CUDA driver (cu121 for driver supporting CUDA 12.2) +uv pip install torch==2.5.1 torchvision==0.20.1 \ + --index-url https://download.pytorch.org/whl/cu121 -# flash-attn pre-built wheel (no compilation needed) +# flash-attn pre-built wheel (Python 3.12 / CUDA 12.x / torch 2.5.1) wget https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl -pip install flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl +uv pip install flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl # HEIF image support -pip install pi-heif +uv pip install pi-heif ``` -> **flash-attn note:** The pre-built wheel above is for Python 3.12, CUDA 12.x, -> torch 2.5.1. If your configuration differs, find the matching wheel at -> https://github.com/Dao-AILab/flash-attention/releases +> If your Python / CUDA / torch combination differs, find the matching +> flash-attn wheel at +> . -### 4. What gets auto-downloaded +### What gets auto-downloaded -When you run `oellm schedule-eval`, the platform automatically pre-downloads -the following on the login node (before SLURM submission, so compute nodes do -not need internet access): +`oellm schedule-eval` pre-downloads the following on the login node so +compute nodes do not need internet access: | Asset | HF repo | Size | |---|---|---| diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 42900f9b..03a20ee3 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -13,7 +13,10 @@ CSV_PATH="{csv_path}" NUM_JOBS={num_jobs} TOTAL_EVALS={total_evals} -LIMIT="{limit}" +# Exported so contrib suite plugins (which spawn their own Python subprocesses +# via oellm.contrib.dispatch) can read it from os.environ. Built-in suites +# below still interpolate $LIMIT directly into their CLI flags. +export LIMIT="{limit}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" diff --git a/oellm/scheduler.py b/oellm/scheduler.py index af203c91..61a80ddd 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -306,7 +306,16 @@ def schedule_evals( logging.warning("No evaluation jobs to schedule.") return None - df["eval_suite"] = df["eval_suite"].str.lower() + # Lowercase the suite name only, preserve any ``:model_flags`` suffix + # verbatim — contrib dispatch keys can be case-sensitive (e.g. + # AudioBench's ``Qwen2-Audio-7B-Instruct`` is matched literally). + def _lower_suite_only(s: str) -> str: + if ":" in s: + head, tail = s.split(":", 1) + return f"{head.lower()}:{tail}" + return s.lower() + + df["eval_suite"] = df["eval_suite"].map(_lower_suite_only) # Ensure that all datasets required by the tasks are cached locally to avoid # network access on compute nodes. diff --git a/pyproject.toml b/pyproject.toml index e8662646..8dcdb8f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,19 @@ audio = [ "librosa", "jiwer", ] +# AudioBench contrib plugin. AudioBench itself is not pip-installable +# (no build backend upstream, bare imports), so AUDIOBENCH_DIR in +# clusters.yaml points at a local git clone and suite.py subprocesses into +# ``python src/main_evaluate.py``. These are our post-processing deps. +audiobench = [ + "jiwer<3", # AudioBench uses jiwer.compute_measures, removed in 3.0 + "transformers>=4.45,<5", # AudioBench's Qwen2-Audio loader uses the v4 processor API (`audios=` kwarg); v5 silently drops audio inputs and produces garbage predictions + "sacrebleu", # BLEU verification (covost2) + "pythainlp", # Thai tokenisation for gigaspeech2_thai + "evaluate", # MMAU / METEOR post-processing + "soundfile", + "librosa", +] [project.scripts] oellm = "oellm.main:main" diff --git a/tests/test_audiobench.py b/tests/test_audiobench.py new file mode 100644 index 00000000..d7617eb7 --- /dev/null +++ b/tests/test_audiobench.py @@ -0,0 +1,946 @@ +"""Tests for the AudioBench contrib benchmark integration. + +The shape of these tests reflects AudioBench's actual upstream API +(``$AUDIOBENCH_DIR/src/main_evaluate.py``), which we discovered while +debugging the first cluster smoke test: + +* ``main()`` accepts only ``dataset_name`` / ``model_name`` / ``metrics`` / + ``overwrite`` / ``number_of_samples`` — no ``--model``, no ``--log_dir``, + no ``--data_dir``. +* ``Model.__init__`` and ``Dataset.load_dataset`` dispatch on **exact** + string match against fixed lists; AudioBench cannot evaluate arbitrary + HF checkpoints (only the variants whose loaders ship under + ``model_src/``), and split selection happens via the dataset_name itself + (``gigaspeech2_thai``, ``spoken-mqa_short_digit``) — there is no + ``--data_dir`` flag. +* AudioBench writes scores to the hardcoded path + ``$cwd/log_for_all_models//__score.json``. +* Without ``--overwrite True`` AudioBench skips evaluation when a stale + score file exists, so :func:`oellm.contrib.audiobench.suite.run` always + passes that flag. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +from oellm.task_groups import ( + _collect_dataset_specs, + _expand_task_groups, + get_all_task_group_names, +) + +SUITE = "audiobench" +TOP_GROUP = "audio-audiobench" +ASR_GROUP = "audio-audiobench-asr" +ST_GROUP = "audio-audiobench-st" +REASONING_GROUP = "audio-audiobench-reasoning" + +# Canonical task names that MUST be in the registry. A silent rename +# breaks the build. +NEW_TASKS = { + # ASR (9) + "audiobench_aishell_asr_zh_test", + "audiobench_earnings21_test", + "audiobench_earnings22_test", + "audiobench_tedlium3_long_form_test", + "audiobench_gigaspeech2_thai", + "audiobench_gigaspeech2_indo", + "audiobench_gigaspeech2_viet", + "audiobench_seame_dev_man", + "audiobench_seame_dev_sge", + # ST (5) + "audiobench_covost2_en_id_test", + "audiobench_covost2_en_ta_test", + "audiobench_covost2_id_en_test", + "audiobench_covost2_zh_en_test", + "audiobench_covost2_ta_en_test", + # Reasoning (6) + "audiobench_spoken_mqa_short_digit", + "audiobench_spoken_mqa_long_digit", + "audiobench_spoken_mqa_single_step_reasoning", + "audiobench_spoken_mqa_multi_step_reasoning", + "audiobench_mmau_mini", + "audiobench_audiocaps_test", +} + +DUAL_TASKS = { + "audiobench_librispeech_test_clean", + "audiobench_librispeech_test_other", + "audiobench_common_voice_15_en_test", + "audiobench_gigaspeech_test", + "audiobench_peoples_speech_test", + "audiobench_tedlium3_test", + "audiobench_covost2_en_zh_test", +} + +ALL_PHASE1_TASKS = NEW_TASKS | DUAL_TASKS + + +# --------------------------------------------------------------------------- +# Registry — task.py +# --------------------------------------------------------------------------- + + +class TestTaskRegistry: + def test_registry_has_exactly_27_tasks(self): + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + assert len(AUDIOBENCH_TASKS) == 27 + + def test_registry_covers_all_phase1_task_names(self): + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + names = {t.name for t in AUDIOBENCH_TASKS} + assert names == ALL_PHASE1_TASKS + + def test_every_task_has_audiobench_prefix(self): + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + for t in AUDIOBENCH_TASKS: + assert t.name.startswith("audiobench_"), t.name + + def test_every_task_has_audiollms_or_amao_hf_repo(self): + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + for t in AUDIOBENCH_TASKS: + assert t.hf_repo.startswith(("AudioLLMs/", "amao0o0/")), ( + f"{t.name} has unexpected repo {t.hf_repo}" + ) + + def test_asr_tasks_all_use_wer(self): + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + for t in AUDIOBENCH_TASKS: + if t.family == "asr": + assert t.metric == "wer", f"{t.name}: {t.metric}" + + def test_st_tasks_all_use_bleu(self): + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + for t in AUDIOBENCH_TASKS: + if t.family == "st": + assert t.metric == "bleu", f"{t.name}: {t.metric}" + + def test_gigaspeech2_tasks_use_per_split_upstream_name(self): + """All 3 GigaSpeech2 tasks share one HF repo, but AudioBench's + ``--dataset_name`` dispatch keys are the split-suffixed forms + (``gigaspeech2_thai``/``_indo``/``_viet``) — there is no + ``--data_dir`` flag. + """ + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + gs2 = [t for t in AUDIOBENCH_TASKS if "gigaspeech2" in t.name] + assert len(gs2) == 3 + assert {t.hf_repo for t in gs2} == {"AudioLLMs/gigaspeech2-test"} + assert {t.upstream_name for t in gs2} == { + "gigaspeech2_thai", + "gigaspeech2_indo", + "gigaspeech2_viet", + } + + def test_spoken_mqa_tasks_use_per_split_upstream_name(self): + """All 4 spoken-mqa tasks share one HF repo; AudioBench dispatches + via the hyphen-prefixed split-suffixed dataset_name + (``spoken-mqa_``). + """ + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + smqa = [t for t in AUDIOBENCH_TASKS if "spoken_mqa" in t.name] + assert len(smqa) == 4 + assert {t.hf_repo for t in smqa} == {"amao0o0/spoken-mqa"} + assert {t.upstream_name for t in smqa} == { + "spoken-mqa_short_digit", + "spoken-mqa_long_digit", + "spoken-mqa_single_step_reasoning", + "spoken-mqa_multi_step_reasoning", + } + + def test_spoken_mqa_uses_acc_metric_upstream(self): + """Upstream metric for spoken-mqa is ``acc`` (the canonical key + we expose externally is ``accuracy``). + """ + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS + + for t in AUDIOBENCH_TASKS: + if "spoken_mqa" in t.name: + assert t.metric == "accuracy" + assert t.upstream_metric == "acc" + + def test_get_task_spec_returns_spec(self): + from oellm.contrib.audiobench.task import get_task_spec + + spec = get_task_spec("audiobench_librispeech_test_clean") + assert spec.upstream_name == "librispeech_test_clean" + assert spec.metric == "wer" + assert spec.family == "asr" + + def test_get_task_spec_unknown_raises(self): + from oellm.contrib.audiobench.task import get_task_spec + + with pytest.raises(KeyError, match="Unknown AudioBench task"): + get_task_spec("audiobench_does_not_exist") + + def test_no_task_spec_carries_data_dir_attribute(self): + """``data_dir`` was removed once we discovered AudioBench has no + such flag; guard against accidental reintroduction. + """ + from oellm.contrib.audiobench.task import AUDIOBENCH_TASKS, AudioBenchTaskSpec + + # Field removed entirely from the dataclass. + assert "data_dir" not in AudioBenchTaskSpec.__dataclass_fields__ + for t in AUDIOBENCH_TASKS: + assert not hasattr(t, "data_dir") + + +# --------------------------------------------------------------------------- +# Adapter — adapter.py +# --------------------------------------------------------------------------- + + +class TestAudioBenchModelAdapter: + """Adapter must return AudioBench's literal ``model_name`` dispatch keys. + + Each pattern check is a regression target — AudioBench's ``model.py`` + does ``if self.model_name == "":`` and raises + NotImplementedError on any other value. + """ + + @pytest.fixture + def adapter_cls(self): + from oellm.contrib.audiobench.adapter import AudioBenchModelAdapter + from oellm.core.base_model_adapter import BaseModelAdapter + + return AudioBenchModelAdapter, BaseModelAdapter + + def test_is_base_model_adapter(self, adapter_cls): + cls, base = adapter_cls + assert issubclass(cls, base) + + def test_qwen2_audio_7b_instruct_returns_literal_key(self, adapter_cls): + cls, _ = adapter_cls + # AudioBench dispatches on the literal "Qwen2-Audio-7B-Instruct". + assert ( + cls("Qwen/Qwen2-Audio-7B-Instruct").to_contrib_flags() + == "Qwen2-Audio-7B-Instruct" + ) + + def test_qwen_audio_chat_returns_literal_key(self, adapter_cls): + cls, _ = adapter_cls + assert cls("Qwen/Qwen-Audio-Chat").to_contrib_flags() == "Qwen-Audio-Chat" + + def test_salmonn_returns_salmonn_7b(self, adapter_cls): + cls, _ = adapter_cls + # AudioBench only ships the 7B variant (model_src/salmonn_7b.py). + assert cls("tsinghua/SALMONN-7B").to_contrib_flags() == "SALMONN_7B" + + def test_whisper_large_v3(self, adapter_cls): + cls, _ = adapter_cls + assert cls("openai/whisper-large-v3").to_contrib_flags() == "whisper_large_v3" + + def test_whisper_large_v2(self, adapter_cls): + cls, _ = adapter_cls + assert cls("openai/whisper-large-v2").to_contrib_flags() == "whisper_large_v2" + + def test_meralion_returns_full_literal_key(self, adapter_cls): + cls, _ = adapter_cls + assert ( + cls("MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION").to_contrib_flags() + == "MERaLiON-AudioLLM-Whisper-SEA-LION" + ) + + def test_phi_4_multimodal(self, adapter_cls): + cls, _ = adapter_cls + assert ( + cls("microsoft/Phi-4-multimodal-instruct").to_contrib_flags() + == "phi_4_multimodal_instruct" + ) + + def test_unknown_returns_none(self, adapter_cls): + """AudioBench has no generic loader. Unmatched paths must return + ``None`` so :func:`suite.run` can raise a clear error rather than + falling through to a fictitious dispatch key. + """ + cls, _ = adapter_cls + assert cls("random/unknown-model").to_contrib_flags() is None + + def test_module_level_detect_function(self): + from oellm.contrib.audiobench.adapter import detect_audiobench_model_type + + assert ( + detect_audiobench_model_type("Qwen/Qwen2-Audio-7B-Instruct") + == "Qwen2-Audio-7B-Instruct" + ) + assert detect_audiobench_model_type("completely/unknown") is None + + +# --------------------------------------------------------------------------- +# Suite plugin protocol — suite.py +# --------------------------------------------------------------------------- + + +class TestSuiteProtocol: + @pytest.fixture + def suite(self): + import oellm.contrib.audiobench.suite as s + + return s + + def test_suite_name(self, suite): + assert suite.SUITE_NAME == "audiobench" + + def test_cluster_env_vars_declared(self, suite): + assert "AUDIOBENCH_DIR" in suite.CLUSTER_ENV_VARS + + def test_task_groups_contains_all_four_groups(self, suite): + groups = suite.TASK_GROUPS["task_groups"] + for g in (TOP_GROUP, ASR_GROUP, ST_GROUP, REASONING_GROUP): + assert g in groups, f"{g} missing from TASK_GROUPS" + + def test_top_level_group_has_all_27_tasks(self, suite): + tasks = suite.TASK_GROUPS["task_groups"][TOP_GROUP]["tasks"] + assert len(tasks) == 27 + + def test_task_metrics_present_for_all_leaves(self, suite): + metrics = suite.TASK_GROUPS["task_metrics"] + assert set(metrics.keys()) == ALL_PHASE1_TASKS + + def test_all_groups_are_zero_shot(self, suite): + for name in (TOP_GROUP, ASR_GROUP, ST_GROUP, REASONING_GROUP): + group = suite.TASK_GROUPS["task_groups"][name] + assert group["n_shots"] == [0] + assert group["suite"] == SUITE + + def test_detect_model_flags_qwen2_audio(self, suite): + assert ( + suite.detect_model_flags("Qwen/Qwen2-Audio-7B-Instruct") + == "Qwen2-Audio-7B-Instruct" + ) + + def test_detect_model_flags_unknown_returns_none(self, suite): + assert suite.detect_model_flags("some/unknown-model") is None + + def test_parse_results_recognises_audiobench_json(self, suite): + data = { + "model_name_or_path": "/path/to/model", + "results": { + "audiobench_librispeech_test_clean": {"wer": 0.047}, + }, + "configs": {"audiobench_librispeech_test_clean": {"num_fewshot": 0}}, + } + result = suite.parse_results(data) + assert result is not None + model_id, task_name, n_shot, metrics = result + assert model_id == "/path/to/model" + assert task_name == "audiobench_librispeech_test_clean" + assert n_shot == 0 + assert metrics["wer"] == pytest.approx(0.047) + + def test_parse_results_rejects_non_audiobench_json(self, suite): + # lmms-eval style — no audiobench_ prefix. + data = { + "model_name_or_path": "some/model", + "results": {"librispeech_test_clean": {"wer,none": 0.05}}, + "configs": {"librispeech_test_clean": {"num_fewshot": 0}}, + } + assert suite.parse_results(data) is None + + def test_parse_results_empty_returns_none(self, suite): + assert suite.parse_results({}) is None + + def test_parse_results_malformed_returns_none(self, suite): + assert suite.parse_results({"results": "not a dict"}) is None + + +# --------------------------------------------------------------------------- +# TASK_GROUPS integration with core registry. +# --------------------------------------------------------------------------- + + +class TestTaskGroupsIntegration: + def test_groups_registered_via_registry(self): + all_names = get_all_task_group_names() + for g in (TOP_GROUP, ASR_GROUP, ST_GROUP, REASONING_GROUP): + assert g in all_names + + def test_top_group_expands_to_27_zero_shot_tasks(self): + results = _expand_task_groups([TOP_GROUP]) + assert len(results) == 27 + for r in results: + assert r.n_shot == 0 + assert r.suite == SUITE + + def test_top_group_expands_to_expected_task_names(self): + results = _expand_task_groups([TOP_GROUP]) + assert {r.task for r in results} == ALL_PHASE1_TASKS + + def test_asr_group_has_15_leaves(self): + results = _expand_task_groups([ASR_GROUP]) + # 9 new ASR + 6 dual ASR = 15. + assert len(results) == 15 + for r in results: + assert r.suite == SUITE + + def test_st_group_has_6_leaves(self): + results = _expand_task_groups([ST_GROUP]) + # 5 new ST + 1 dual (en→zh) = 6. + assert len(results) == 6 + + def test_reasoning_group_has_6_leaves(self): + results = _expand_task_groups([REASONING_GROUP]) + # 4 spoken-mqa + mmau_mini + audiocaps = 6. + assert len(results) == 6 + + def test_dataset_specs_flag_snapshot_download(self): + # Auto-derived from the ``audio-*`` group-name prefix in + # _collect_dataset_specs. + specs = _collect_dataset_specs([TOP_GROUP]) + assert specs, "No dataset specs returned" + for s in specs: + assert s.needs_snapshot_download, ( + f"DatasetSpec for {s.repo_id} missing needs_snapshot_download=True" + ) + + def test_dataset_specs_dedupe_shared_repos(self): + # gigaspeech2 (3 tasks) → 1 spec; spoken-mqa (4 tasks) → 1 spec. + specs = _collect_dataset_specs([TOP_GROUP]) + repo_ids = [s.repo_id for s in specs] + assert repo_ids.count("AudioLLMs/gigaspeech2-test") == 1 + assert repo_ids.count("amao0o0/spoken-mqa") == 1 + + def test_dataset_specs_contain_audiollms_repos(self): + specs = _collect_dataset_specs([TOP_GROUP]) + repo_ids = {s.repo_id for s in specs} + assert "AudioLLMs/librispeech_test_clean" in repo_ids + assert "AudioLLMs/earnings21_test" in repo_ids + assert "AudioLLMs/MMAU-mini" in repo_ids + assert "amao0o0/spoken-mqa" in repo_ids + + +# --------------------------------------------------------------------------- +# Registry auto-discovery. +# --------------------------------------------------------------------------- + + +class TestRegistryDiscovery: + def test_audiobench_suite_is_auto_discovered(self): + # Clear the _discover() cache so this test doesn't rely on import + # order from earlier tests. + from oellm import registry + + registry._discover.cache_clear() + mod = registry.get_suite("audiobench") + assert mod.SUITE_NAME == "audiobench" + assert hasattr(mod, "run") + assert hasattr(mod, "parse_results") + assert hasattr(mod, "detect_model_flags") + + def test_task_groups_merged_into_registry(self): + from oellm import registry + + registry._discover.cache_clear() + merged = registry.get_all_task_groups() + assert TOP_GROUP in merged["task_groups"] + assert "audiobench_librispeech_test_clean" in merged["task_metrics"] + + +# --------------------------------------------------------------------------- +# EvalRunner — resolve_suite wires audiobench through the adapter. +# --------------------------------------------------------------------------- + + +class TestRunnerIntegration: + def test_resolve_suite_appends_audiobench_dispatch_key(self): + from oellm.constants import EvaluationJob + from oellm.runner import EvalRunner + + runner = EvalRunner() + job = EvaluationJob( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task_path="audiobench_librispeech_test_clean", + n_shot=0, + eval_suite="audiobench", + ) + result = runner.resolve_suite(job) + # AudioBench's literal dispatch key (case-sensitive) must come + # through verbatim so dispatch.py / suite.run get the exact value + # AudioBench's ``Model`` class compares against. + assert result == "audiobench:Qwen2-Audio-7B-Instruct" + + def test_resolve_suite_unknown_model_passes_through_bare(self): + """When the adapter returns ``None`` (no AudioBench-supported + loader for the model path), ``resolve_suite`` keeps the bare + suite name; :func:`suite.run` then raises a clear error at + dispatch time rather than fabricating a fake key. + """ + from oellm.constants import EvaluationJob + from oellm.runner import EvalRunner + + runner = EvalRunner() + job = EvaluationJob( + model_path="some/unknown-model", + task_path="audiobench_mmau_mini", + n_shot=0, + eval_suite="audiobench", + ) + result = runner.resolve_suite(job) + assert result == "audiobench" # bare, no ``:flags`` suffix + + +# --------------------------------------------------------------------------- +# run() subprocess harness — exercise with a mocked subprocess. +# --------------------------------------------------------------------------- + + +class TestRunHarness: + """Exercise suite.run() with a mocked subprocess, verifying both the + CLI it would invoke (matching AudioBench's actual ``main()`` signature) + and that we read the score file from AudioBench's hardcoded output + location. + """ + + def _fake_audiobench_tree(self, tmp_path: Path) -> Path: + """Create a minimal directory tree that looks like an AudioBench clone.""" + ab_dir = tmp_path / "AudioBench" + (ab_dir / "src").mkdir(parents=True) + (ab_dir / "src" / "main_evaluate.py").write_text("# placeholder\n") + return ab_dir + + @staticmethod + def _score_file_path( + ab_dir: Path, model_name: str, dataset: str, metric: str + ) -> Path: + """Mirror suite._extract_metrics' path construction.""" + return ( + ab_dir / "log_for_all_models" / model_name / f"{dataset}_{metric}_score.json" + ) + + def _fake_run_writing_score( + self, ab_dir: Path, *, score_value: float, body_shape: str = "flat" + ): + """Build a fake_run that writes a score file at AudioBench's + hardcoded path, parameterized by the JSON shape we want to test. + """ + + def fake_run(cmd, cwd, env, check): + model_name = cmd[cmd.index("--model_name") + 1] + dataset = cmd[cmd.index("--dataset_name") + 1] + metric = cmd[cmd.index("--metrics") + 1] + score_file = self._score_file_path(Path(cwd), model_name, dataset, metric) + score_file.parent.mkdir(parents=True, exist_ok=True) + if body_shape == "flat": + score_file.write_text(json.dumps({metric: score_value})) + elif body_shape == "nested": + score_file.write_text( + json.dumps({"metrics": {metric: {"score": score_value, "n": 100}}}) + ) + elif body_shape == "missing_metric": + score_file.write_text(json.dumps({"irrelevant": 1})) + elif body_shape == "no_file": + pass # deliberately don't write + else: + raise ValueError(f"unknown body_shape: {body_shape}") + return _FakeCompletedProcess(0) + + return fake_run + + def test_run_missing_audiobench_dir_raises(self, tmp_path): + from oellm.contrib.audiobench.suite import run + + with pytest.raises(RuntimeError, match="AUDIOBENCH_DIR must be set"): + run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags="Qwen2-Audio-7B-Instruct", + env={}, # no AUDIOBENCH_DIR + ) + + def test_run_missing_entrypoint_raises(self, tmp_path): + from oellm.contrib.audiobench.suite import run + + bad_dir = tmp_path / "not-audiobench" + bad_dir.mkdir() + with pytest.raises(FileNotFoundError, match="AudioBench entry point"): + run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(bad_dir)}, + ) + + def test_run_unmapped_model_raises(self, tmp_path): + """AudioBench has no generic loader. When ``model_flags`` is + ``None`` (adapter found no match), :func:`run` must fail loudly + rather than invoking AudioBench with a missing/empty model_name. + """ + from oellm.contrib.audiobench.suite import run + + ab_dir = self._fake_audiobench_tree(tmp_path) + with pytest.raises(RuntimeError, match="Could not map model_path"): + run( + model_path="random/unknown-model", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags=None, + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + def test_run_invokes_subprocess_with_expected_cli(self, tmp_path): + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score(ab_dir, score_value=0.063), + ) as mock_sp: + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir), "LIMIT": "100"}, + ) + + assert mock_sp.call_count == 1 + cmd = mock_sp.call_args.args[0] + assert cmd[:2] == ["python", "src/main_evaluate.py"] + + # AudioBench's actual main() signature: dataset_name / model_name + # / metrics / overwrite / number_of_samples. No --model, no + # --log_dir, no --data_dir. + assert cmd[cmd.index("--dataset_name") + 1] == "librispeech_test_clean" + assert cmd[cmd.index("--model_name") + 1] == "Qwen2-Audio-7B-Instruct" + assert cmd[cmd.index("--metrics") + 1] == "wer" + assert cmd[cmd.index("--overwrite") + 1] == "True" + assert cmd[cmd.index("--number_of_samples") + 1] == "100" + + # Flags AudioBench does NOT accept must not be in the cmd. + assert "--model" not in cmd # only --model_name exists upstream + assert "--log_dir" not in cmd # AudioBench writes to a fixed path + assert "--data_dir" not in cmd # split selection is via dataset_name + + # cwd is AUDIOBENCH_DIR so AudioBench's relative writes + # (log_for_all_models/...) land inside the clone. + assert mock_sp.call_args.kwargs["cwd"] == str(ab_dir) + + # Output JSON is lmms-eval-shaped. + body = json.loads(output_path.read_text()) + assert body["model_name_or_path"] == "Qwen/Qwen2-Audio-7B-Instruct" + assert body["results"]["audiobench_librispeech_test_clean"][ + "wer" + ] == pytest.approx(0.063) + assert body["configs"]["audiobench_librispeech_test_clean"]["num_fewshot"] == 0 + + def test_run_uses_per_split_dataset_name_for_gigaspeech2(self, tmp_path): + """GigaSpeech2 splits are dispatched via the dataset_name itself + (``gigaspeech2_thai``), not a ``--data_dir`` flag. + """ + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score(ab_dir, score_value=0.12), + ) as mock_sp: + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_gigaspeech2_thai", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + cmd = mock_sp.call_args.args[0] + assert cmd[cmd.index("--dataset_name") + 1] == "gigaspeech2_thai" + assert "--data_dir" not in cmd + + def test_run_omits_number_of_samples_when_limit_empty(self, tmp_path): + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score(ab_dir, score_value=0.1), + ) as mock_sp: + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir), "LIMIT": ""}, + ) + + cmd = mock_sp.call_args.args[0] + assert "--number_of_samples" not in cmd + + def test_run_always_passes_overwrite_true(self, tmp_path): + """AudioBench skips evaluation when a stale score file already + exists unless ``--overwrite True`` is passed; we always pass it + because we do our own deduplication via output_path. + """ + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score(ab_dir, score_value=0.1), + ) as mock_sp: + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + cmd = mock_sp.call_args.args[0] + assert cmd[cmd.index("--overwrite") + 1] == "True" + + def test_run_nonzero_exit_raises(self, tmp_path): + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + return_value=_FakeCompletedProcess(1), + ): + with pytest.raises(RuntimeError, match="AudioBench exited with code 1"): + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + def test_run_handles_nested_metric_json(self, tmp_path): + """AudioBench's score-file shape has drifted across releases; we + tolerate both ``{"wer": 0.05}`` and + ``{"metrics": {"wer": {"score": 0.05}}}`` layouts. + """ + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score( + ab_dir, score_value=0.051, body_shape="nested" + ), + ): + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + body = json.loads(output_path.read_text()) + assert body["results"]["audiobench_librispeech_test_clean"][ + "wer" + ] == pytest.approx(0.051) + + def test_run_missing_score_file_raises(self, tmp_path): + """If AudioBench exits 0 but doesn't write the score file at the + expected path, surface a clear error rather than producing an + empty CSV row downstream. + """ + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score( + ab_dir, score_value=0.0, body_shape="no_file" + ), + ): + with pytest.raises( + RuntimeError, match="AudioBench did not write expected score file" + ): + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + def test_run_score_file_without_metric_key_raises(self, tmp_path): + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + output_path = tmp_path / "result.json" + + with patch( + "oellm.contrib.audiobench.suite.subprocess.run", + side_effect=self._fake_run_writing_score( + ab_dir, score_value=0.0, body_shape="missing_metric" + ), + ): + with pytest.raises(RuntimeError, match="Could not locate metric"): + suite.run( + model_path="Qwen/Qwen2-Audio-7B-Instruct", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=output_path, + model_flags="Qwen2-Audio-7B-Instruct", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + + +class _FakeCompletedProcess: + """Stand-in for subprocess.CompletedProcess.""" + + def __init__(self, returncode: int) -> None: + self.returncode = returncode + + +# --------------------------------------------------------------------------- +# schedule_evals dry-run — wiring smoke test. +# --------------------------------------------------------------------------- + + +class TestScheduleEvalsDryRun: + def test_dry_run_writes_audiobench_suite_to_csv(self, tmp_path): + import pandas as pd + + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="Qwen/Qwen2-Audio-7B-Instruct", + task_groups=ASR_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + csv_files = list(tmp_path.glob("**/jobs.csv")) + assert len(csv_files) == 1 + df = pd.read_csv(csv_files[0]) + # All rows route to audiobench (with model-flag suffix). + assert all(s.startswith("audiobench") for s in df["eval_suite"].unique()) + # task_path column contains canonical audiobench_ names. + assert all(t.startswith("audiobench_") for t in df["task_path"].unique()) + + def test_dry_run_preserves_model_flag_capitalization(self, tmp_path): + """Regression: scheduler.py used to lowercase the entire eval_suite + column, breaking AudioBench's case-sensitive dispatch keys + (``Qwen2-Audio-7B-Instruct`` was being mangled to + ``qwen2-audio-7b-instruct``). + """ + import pandas as pd + + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="Qwen/Qwen2-Audio-7B-Instruct", + task_groups=ASR_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + csv_files = list(tmp_path.glob("**/jobs.csv")) + df = pd.read_csv(csv_files[0]) + suites = set(df["eval_suite"].unique()) + # The exact AudioBench dispatch literal must come through case-intact. + assert "audiobench:Qwen2-Audio-7B-Instruct" in suites + + def test_dry_run_sbatch_contains_contrib_dispatch(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="Qwen/Qwen2-Audio-7B-Instruct", + task_groups=TOP_GROUP, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + content = sbatch_files[0].read_text() + assert "oellm.contrib.dispatch" in content + # LIMIT is exported so contrib plugins can read it. + assert "export LIMIT=" in content + + +# --------------------------------------------------------------------------- +# collect_results compatibility — verify a run() output flows through unchanged. +# --------------------------------------------------------------------------- + + +class TestCollectResultsCompat: + def test_collect_results_parses_audiobench_json(self, tmp_path): + import pandas as pd + + from oellm.main import collect_results + + results_dir = tmp_path / "results" + results_dir.mkdir() + + mock_output = { + "model_name_or_path": "/cluster/models/Qwen2-Audio-7B", + "results": { + "audiobench_librispeech_test_clean": {"wer": 0.052}, + }, + "configs": {"audiobench_librispeech_test_clean": {"num_fewshot": 0}}, + } + (results_dir / "ab123.json").write_text(json.dumps(mock_output)) + + output_csv = str(tmp_path / "results.csv") + collect_results(str(tmp_path), output_csv=output_csv) + + df = pd.read_csv(output_csv) + assert len(df) == 1 + row = df.iloc[0] + assert row["task"] == "audiobench_librispeech_test_clean" + assert float(row["performance"]) == pytest.approx(0.052) + assert row["model_name"] == "/cluster/models/Qwen2-Audio-7B" From a72abbb2b9db8bbb80cf090e8ed016c7d19fd04b Mon Sep 17 00:00:00 2001 From: islobozhan Date: Sun, 17 May 2026 17:47:33 +0200 Subject: [PATCH 23/44] [Base] Metric correctness and orchestration fixes Metric correctness and orchestration fixes --- docs/CONTAINERS.md | 6 +- docs/VENV.md | 60 ++--- oellm/constants.py | 26 ++ oellm/main.py | 8 + oellm/resources/task-groups.yaml | 41 +++- oellm/resources/template.sbatch | 85 +++---- oellm/results.py | 139 +++++++++-- oellm/scheduler.py | 11 + oellm/task_groups.py | 55 +++-- oellm/utils.py | 318 +++++++++++++++++-------- pyproject.toml | 72 +++++- requirements-venv-dclm.txt | 10 - requirements-venv-evalchemy.txt | 15 -- requirements-venv.txt | 21 -- tests/test_collect_results.py | 56 ++++- tests/test_reporter.py | 51 +++- tests/test_schedule_evals.py | 4 + tests/test_utils.py | 393 ++++++++++++++++++++++++++++++- tests/test_video_task_groups.py | 40 +++- 19 files changed, 1128 insertions(+), 283 deletions(-) delete mode 100644 requirements-venv-dclm.txt delete mode 100644 requirements-venv-evalchemy.txt delete mode 100644 requirements-venv.txt diff --git a/docs/CONTAINERS.md b/docs/CONTAINERS.md index 31d6e7c9..e2afbd94 100644 --- a/docs/CONTAINERS.md +++ b/docs/CONTAINERS.md @@ -18,7 +18,11 @@ Images are compressed with zstd (level 3) via mksquashfs for a good balance of s Image benchmarks (`suite: lmms_eval`) require `lmms-eval` to be available in the execution environment. There are two ways to provide it: **Option 1 — Custom venv (recommended for development):** -Install `lmms-eval` via `requirements-venv.txt` and pass `--venv-path` to the CLI. See [VENV.md](VENV.md) for setup instructions. +Install `lmms-eval` via the `[image]` (or `[audio]`/`[video]`) pyproject extra and pass `--venv-path` to the CLI: +```bash +uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' +``` +See [VENV.md](VENV.md) for full setup instructions. **Option 2 — Container with lmms-eval:** Build a container `.def` file that includes `lmms-eval` alongside `lm-eval`: diff --git a/docs/VENV.md b/docs/VENV.md index da9f53bb..beab9c7d 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -28,25 +28,30 @@ Use `oellm list-tasks` to see which suite a given task group routes to. ## Setup (general venv) -1. Create a venv with Python 3.12: - ```bash - uv venv --python 3.12 /path/to/.venv - ``` - -2. Install lm-eval and lmms-eval dependencies: - ```bash - uv pip install --python /path/to/.venv/bin/python -r requirements-venv.txt - ``` - - This installs `lm-eval`, `torch`, `transformers`, `accelerate`, `datasets<4.0.0`, and `lmms-eval`. +```bash +# 1. Create venv +uv venv --python 3.12 /path/to/.venv + +# 2. Install lmms-eval editable from main +# (Editable is required — wheel build drops `_default_template_yaml` files.) +uv pip install --python /path/to/.venv/bin/python \ + -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git#egg=lmms-eval" + +# 3. Install oellm-cli with engine extras +uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' + +# 4. Install lighteval as an isolated uv tool (datasets version conflict) +UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ + uv tool install --python 3.12 \ + --with "langcodes[data]" --with "pillow" \ + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" +``` -3. Install lighteval as isolated tool (avoids datasets version conflict): - ```bash - UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ - uv tool install --python 3.12 \ - --with "langcodes[data]" --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" - ``` +Verify with: +```bash +/path/to/.venv/bin/python -c \ + 'from lmms_eval.tasks import TaskManager; TaskManager("INFO"); print("OK")' +``` ## Usage @@ -71,20 +76,21 @@ lm-eval requires `datasets<4.0.0` while lighteval requires `datasets>=4.0.0`. In ## Dependency Summary -| Package | Install method | Reason | +| Component | Install | Reason | |---|---|---| -| `lm-eval`, `torch`, `transformers`, `accelerate`, `datasets<4.0.0`, `lmms-eval` | `uv pip install -r requirements-venv.txt` | lm-eval + image eval, compatible dataset pin | -| `lighteval[multilingual]` | `uv tool install` (isolated) | Requires `datasets>=4.0.0` — must be isolated | - -lm-eval requires `datasets<4.0.0` while lighteval requires `datasets>=4.0.0`. Installing lighteval as an isolated uv tool (like the containers do) avoids this conflict. +| `lmms-eval` | `uv pip install -e "git+https://…@#egg=lmms-eval"` | Editable: wheel drops template files | +| `text`, `image`, `audio` extras | `uv pip install '.[text,image,audio]'` | lm-eval + transformers pin + audio helpers | +| `lighteval` | `uv tool install …` (isolated) | Needs `datasets>=4.0.0`; conflicts with lm-eval | +| `evalchemy` | `uv pip install '.[evalchemy]'` (own venv) | Forked lm-eval | +| `dclm` | `uv pip install '.[dclm]'` (own venv) | Pinned `lm-eval==0.4.9.2` | ## DCLM-core-22 -`dclm-core-22` needs `lm-eval==0.4.9.2` (v0.4.10+ breaks `agieval_lsat_ar` in few-shot). Use `requirements-venv-dclm.txt` instead of the default requirements: +`dclm-core-22` needs `lm-eval==0.4.9.2` (v0.4.10+ breaks `agieval_lsat_ar` in few-shot). Install in its own venv using the `[dclm]` extra: ```bash uv venv --python 3.12 dclm-core-venv -uv pip install --python dclm-core-venv/bin/python -r requirements-venv-dclm.txt +uv pip install --python dclm-core-venv/bin/python -e '.[dclm]' ``` ```bash @@ -109,10 +115,10 @@ We use [Ali's fork](https://github.com/Ali-Elganzory/evalchemy) which includes a cd evalchemy && git checkout 54ac97648230c4c3a22c3a2b93068b5a4e573f8d && cd .. ``` -2. Create a venv and install dependencies: +2. Create a venv and install dependencies using the `[evalchemy]` extra: ```bash uv venv --python 3.12 evalchemy-venv - uv pip install --python evalchemy-venv/bin/python -r requirements-venv-evalchemy.txt + uv pip install --python evalchemy-venv/bin/python -e '.[evalchemy]' ``` 3. Run with `EVALCHEMY_DIR` pointing to the cloned repo: diff --git a/oellm/constants.py b/oellm/constants.py index e1bf391f..2478f3aa 100644 --- a/oellm/constants.py +++ b/oellm/constants.py @@ -53,6 +53,32 @@ class EvaluationJob: "exact_match", ] +# Tasks whose primary metric is computed by an LLM judge / extractor and +# requires ``OPENAI_API_KEY`` (or compatible) at evaluation time. Pre-flight +# check refuses to schedule these without the key unless +# ``--allow-missing-judge`` is passed. +# +# NOT included: mmbench_en_dev (regex extraction primary; GPT fallback) and +# mathvista_testmini_* (quick_extract regex primary; GPT fallback) — these +# emit valid numbers without a key, so blocking them would cause friction. +# Add tasks here when the metric is genuinely undefined without a judge call. +JUDGE_REQUIRED_TASKS: frozenset[str] = frozenset( + { + # Video + "activitynetqa", + # Audio — AudioBench / VoiceBench judge-graded tasks (`gpt_eval` or + # `llm_as_judge_eval` is the only metric path; no regex fallback). + "alpaca_audio", + "openhermes", + "wavcaps", + "air_bench_chat_sound", + "air_bench_chat_music", + "air_bench_chat_speech", + "air_bench_chat_mixed", + "voicebench_commoneval", + } +) + def detect_lmms_model_type(model_path: str) -> str: """Detect the lmms-eval adapter class name from a model path or HF repo name. diff --git a/oellm/main.py b/oellm/main.py index c1c14698..a6a7ef62 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -47,6 +47,7 @@ def schedule_evals( lm_eval_include_path: str | None = None, local: bool = False, slurm_template_var: str | None = None, + allow_missing_judge: bool = False, ) -> None: """Schedule evaluation jobs for a given set of models, tasks, and number of shots. @@ -88,6 +89,10 @@ def schedule_evals( slurm_template_var: JSON object of template variable overrides. Use exact env var names (PARTITION, ACCOUNT, GPUS_PER_NODE, SLURM_MEM). "TIME" overrides the time limit. Example: '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00","GPUS_PER_NODE":2,"SLURM_MEM":"96G"}' + allow_missing_judge: If True, allow scheduling tasks that need an LLM judge + (e.g. activitynetqa, AudioBench chat-style tasks) when ``OPENAI_API_KEY`` + is not set. Those tasks will emit null performance values. Default False — + strict pre-flight refuses to schedule without the key. """ from oellm.scheduler import schedule_evals as _sched @@ -141,6 +146,7 @@ def schedule_evals( lm_eval_include_path=cfg.lm_eval_include_path, local=cfg.local, slurm_template_var=cfg.slurm_template_var_json, + allow_missing_judge=allow_missing_judge, ) @@ -287,6 +293,7 @@ def eval_command( lm_eval_include_path: str | None = None, local: bool = False, slurm_template_var: str | None = None, + allow_missing_judge: bool = False, ) -> None: """Run evaluations from a YAML config file. @@ -329,6 +336,7 @@ def eval_command( lm_eval_include_path=lm_eval_include_path, local=local, slurm_template_var=slurm_template_var, + allow_missing_judge=allow_missing_judge, ) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index ec848b5e..0fc71c7c 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -47,7 +47,9 @@ task_metrics: bigbench_operators_generate_until: exact_match bigbench_repeat_copy_logic_generate_until: exact_match bigbench_cs_algorithms_generate_until: exact_match - # lmms-eval image benchmark metrics + # lmms-eval image benchmark metrics. + # NOTE: `exact_match` for vqav2/textvqa is a lmms-eval misnomer — it's + # the soft VQA accuracy `min(matching/3, 1)`, not strict equality. vqav2_val: exact_match mme: mme_cognition_score mmbench_en_dev: gpt_eval_score @@ -56,18 +58,21 @@ task_metrics: docvqa_val: anls textvqa_val: exact_match ocrbench: ocrbench_accuracy - # MathVista uses LLM-as-judge; null without judge LLM configured. The - # top-level `mathvista_testmini` is a GROUP that expands to the three - # leaves below — schedule the leaves, not the group. + # MathVista: schedule the 3 leaves (cot / format / solution), not the group. mathvista_testmini_cot: llm_as_judge_eval mathvista_testmini_format: llm_as_judge_eval mathvista_testmini_solution: llm_as_judge_eval - # lmms-eval video benchmark metrics. `video_mmmu` upstream is a GROUP - # that expands to perception / comprehension / adaptation leaves. + # lmms-eval video benchmark metrics. `video_mmmu` is a group of 3 leaves. video_mmmu_perception: mmmu_acc video_mmmu_comprehension: mmmu_acc video_mmmu_adaptation: mmmu_acc - egoschema: submission + # EgoSchema: use `egoschema_subset` (val split with ground truth). + # The bare `egoschema` task is Kaggle-submission-only. + egoschema_subset: score + # MVBench: `mvbench` is a group of 20 subtasks; collect_results + # aggregates per-subtask `mvbench_accuracy` into a synthetic row. + mvbench: mvbench_accuracy + # `videomme_perception_score` IS the overall accuracy despite the name. videomme: videomme_perception_score # ActivityNet-QA requires GPT API access for evaluation (LLM-as-judge) activitynetqa: gpt_eval_accuracy @@ -567,7 +572,7 @@ task_groups: # ── Video Modality (lmms-eval) ──────────────────────────────────────────── video-understanding: - description: "Video understanding benchmarks via lmms-eval (VideoMMMU, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench)" + description: "Video understanding benchmarks via lmms-eval (VideoMMMU, MVBench, EgoSchema-subset, VideoMME, ActivityNet-QA, LongVideoBench)" suite: lmms_eval n_shots: [0] tasks: @@ -577,7 +582,11 @@ task_groups: dataset: lmms-lab/VideoMMMU - task: video_mmmu_adaptation dataset: lmms-lab/VideoMMMU - - task: egoschema + - task: mvbench + dataset: OpenGVLab/MVBench + # MVBench parquet on `main`, .mp4 files on `video` — fetch both. + revisions: [main, video] + - task: egoschema_subset dataset: lmms-lab/egoschema - task: videomme dataset: lmms-lab/Video-MME @@ -597,12 +606,22 @@ task_groups: - task: video_mmmu_comprehension - task: video_mmmu_adaptation + video-mvbench: + description: "MVBench short-clip temporal video understanding via lmms-eval (group of 20 subtasks; mean accuracy)" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mvbench + dataset: OpenGVLab/MVBench + # MVBench parquet on `main`, .mp4 files on `video` — fetch both. + revisions: [main, video] + video-egoschema: - description: "EgoSchema long-form egocentric video QA via lmms-eval" + description: "EgoSchema long-form egocentric video QA via lmms-eval (500-q validation subset with ground truth)" suite: lmms_eval n_shots: [0] tasks: - - task: egoschema + - task: egoschema_subset dataset: lmms-lab/egoschema video-videomme: diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 03a20ee3..232a526b 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -20,10 +20,17 @@ export LIMIT="{limit}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" -# avoiding crashes due to compute nodes not having access to the internet +# Compute nodes are air-gapped — every dataset must be cache-resolved. export HF_HOME=$HF_HOME export HF_DATASETS_CACHE="$HF_HOME/datasets" export HF_HUB_OFFLINE={hf_hub_offline} +# HF_DATASETS_OFFLINE covers `dl_manager.download(arbitrary_url)` (external +# image/video hosts) which HF_HUB_OFFLINE does not. +export HF_DATASETS_OFFLINE={hf_hub_offline} +# lmms-eval auto-redirects "remote" (GPFS/Lustre) caches to a node-local +# /tmp by default; set this to force it to use the shared cache instead. +# See lmms_eval/api/task.py::_resolve_hf_datasets_cache_dir. +export LMMS_EVAL_DATASETS_CACHE="$HF_HOME/datasets" # Path to the shared Singularity image that contains all runtime deps (container mode) export EVAL_SIF_PATH="$EVAL_BASE_DIR/$EVAL_CONTAINER_IMAGE" @@ -61,11 +68,12 @@ fi tail -n +$((START_INDEX + 1)) "$CSV_PATH" | head -n $((END_INDEX - START_INDEX + 1)) | \ while IFS=, read -r model_path task_path n_shot eval_suite do - # Remove trailing carriage returns if script is edited on Windows - model_path=$(echo "$model_path" | tr -d '\r') - task_path=$(echo "$task_path" | tr -d '\r') - n_shot=$(echo "$n_shot" | tr -d '\r') - eval_suite=$(echo "${{eval_suite:-lm_eval}}" | tr -d '\r') + # Strip CR (Windows line endings) from every field. + model_path=${{model_path%$'\r'}} + task_path=${{task_path%$'\r'}} + n_shot=${{n_shot%$'\r'}} + eval_suite=${{eval_suite:-lm_eval}} + eval_suite=${{eval_suite%$'\r'}} # Skip empty lines if [ -z "$model_path" ]; then @@ -103,7 +111,7 @@ do JOB_HOME="$EVAL_BASE_DIR/$USER/container_home/$SLURM_JOB_ID" mkdir -p "$JOB_HOME" SINGULARITY_HOME_ARG="--home $JOB_HOME" - SINGULARITY_ENV_ARGS="--env HF_HOME=$HF_HOME --env HF_DATASETS_CACHE=$HF_DATASETS_CACHE --env HF_HUB_OFFLINE=$HF_HUB_OFFLINE --env TRANSFORMERS_OFFLINE=$TRANSFORMERS_OFFLINE --env NLTK_DATA=$NLTK_DATA" + SINGULARITY_ENV_ARGS="--env HF_HOME=$HF_HOME --env HF_DATASETS_CACHE=$HF_DATASETS_CACHE --env HF_HUB_OFFLINE=$HF_HUB_OFFLINE --env HF_DATASETS_OFFLINE=$HF_DATASETS_OFFLINE --env TRANSFORMERS_OFFLINE=$TRANSFORMERS_OFFLINE --env NLTK_DATA=$NLTK_DATA --env LMMS_EVAL_DATASETS_CACHE=$LMMS_EVAL_DATASETS_CACHE" fi GPU_DEVICES=$(seq -s, 0 $(($GPUS_PER_NODE - 1))) @@ -203,27 +211,21 @@ do _lmms_extra_args=",model_name=$(basename "$model_path")" fi - if [ -n "$VENV_PATH" ]; then - source "$VENV_PATH/bin/activate" - python -m lmms_eval \ - --model "$_lmms_adapter" \ - --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args" \ - --tasks "$task_path" \ - --num_fewshot "$n_shot" \ - --output_path "$OUTPUT_JSON" \ - ${{LIMIT:+--limit $LIMIT}} - else - singularity exec $SINGULARITY_ARGS \ - --bind $BIND_PATHS \ - $EVAL_SIF_PATH \ - python -m lmms_eval \ - --model "$_lmms_adapter" \ - --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args" \ - --tasks "$task_path" \ - --num_fewshot "$n_shot" \ - --output_path "$OUTPUT_JSON" \ - ${{LIMIT:+--limit $LIMIT}} + # Qwen2-VL / Qwen2.5-VL: cap frames-per-video to fit on a 64GB + # A100. lmms-eval defaults to 32 which OOMs on long-form video + # (EgoSchema, VideoMME). Image tasks ignore this flag. + # Override via MAX_NUM_FRAMES env var on larger GPUs. + if [[ "$_lmms_adapter" == "qwen2_vl" || "$_lmms_adapter" == "qwen2_5_vl" ]]; then + _lmms_extra_args+=",max_num_frames=${{MAX_NUM_FRAMES:-8}}" fi + + run_python -m lmms_eval \ + --model "$_lmms_adapter" \ + --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args" \ + --tasks "$task_path" \ + --num_fewshot "$n_shot" \ + --output_path "$OUTPUT_JSON" \ + ${{LIMIT:+--limit $LIMIT}} ;; evalchemy) EVALCHEMY_WORK_DIR="{evalchemy_dir}" @@ -246,30 +248,13 @@ do ;; *) # Contrib suite: dispatch to the Python plugin registry. - # This single case handles ALL registered contrib benchmarks — no - # further template.sbatch changes are needed when adding new ones. - # The suite module's run() writes a lmms-eval-compatible JSON file. _CONTRIB_OUTPUT="{evals_dir}/$(openssl rand -hex 5).json" - - if [ -n "$VENV_PATH" ]; then - source "$VENV_PATH/bin/activate" - python -m oellm.contrib.dispatch \ - --suite "$eval_suite" \ - --model_path "$model_path" \ - --task "$task_path" \ - --n_shot "$n_shot" \ - --output_path "$_CONTRIB_OUTPUT" - else - singularity exec $SINGULARITY_ARGS \ - --bind $BIND_PATHS \ - $EVAL_SIF_PATH \ - python -m oellm.contrib.dispatch \ - --suite "$eval_suite" \ - --model_path "$model_path" \ - --task "$task_path" \ - --n_shot "$n_shot" \ - --output_path "$_CONTRIB_OUTPUT" - fi + run_python -m oellm.contrib.dispatch \ + --suite "$eval_suite" \ + --model_path "$model_path" \ + --task "$task_path" \ + --n_shot "$n_shot" \ + --output_path "$_CONTRIB_OUTPUT" ;; esac diff --git a/oellm/results.py b/oellm/results.py index 2ed66a76..46e13c91 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -14,6 +14,62 @@ from oellm.constants import METRIC_FALLBACK_KEYS from oellm.utils import _setup_logging +# Native scale (max value) of each lmms-eval / lm-eval metric. +# Used to normalize all reported metrics to 0–100 for cross-benchmark +# comparison in the Markdown report and JSON envelope. Whenever a new +# metric is added to ``task_metrics`` in ``task-groups.yaml``, also add +# its native scale here so the normalized column renders correctly. +METRIC_NATIVE_SCALE: dict[str, float] = { + # ── 0–1 scale ── + "exact_match": 1.0, + "acc": 1.0, + "acc_norm": 1.0, + "accuracy": 1.0, + "mmmu_acc": 1.0, + "relaxed_overall": 1.0, + "relaxed_human_split": 1.0, + "relaxed_augmented_split": 1.0, + "anls": 1.0, + "ocrbench_accuracy": 1.0, + "lvb_acc": 1.0, + "score": 1.0, + "wer": 1.0, + "mer": 1.0, + "f1": 1.0, + "semantic_match": 1.0, + # ── 0–100 scale (no scaling needed) ── + "gpt_eval_score": 100.0, + "llm_as_judge_eval": 100.0, + "mvbench_accuracy": 100.0, + "videomme_perception_score": 100.0, + "gpt_eval_accuracy": 100.0, + "submission": 100.0, + "bleu": 100.0, + # ── 0–5 Likert scale (GPT-judge style) ── + "gpt_eval": 5.0, + # ── Unbounded / non-standard ── + # MME emits raw point sums: cognition is /800 (4 categories × 200), + # perception is /2000 (10 categories × 200). See + # lmms_eval/tasks/mme/utils.py::mme_aggregate_results. + "mme_cognition_score": 800.0, + "mme_perception_score": 2000.0, +} + + +def _normalize_to_100(value: float | None, metric_name: str | None) -> float | None: + """Normalize a metric value to a 0–100 scale for cross-benchmark display. + + Returns ``None`` when the metric's native scale is unknown — caller + should fall back to the raw value rather than guess. + """ + if value is None or metric_name is None: + return None + clean = metric_name.split(",")[0] + scale = METRIC_NATIVE_SCALE.get(clean) + if scale is None: + return None + return value * (100.0 / scale) + def _resolve_metric( task_name: str, result_dict: dict, task_metrics: dict @@ -284,6 +340,9 @@ def collect_results( "task": group_name, "n_shot": n_shot, "performance": performance, + "performance_normalized": _normalize_to_100( + performance, metric_name + ), "metric_name": metric_name if metric_name is not None else "", } ) @@ -319,13 +378,32 @@ def collect_results( if n_shot == "unknown" and parsed_n is not None: n_shot = parsed_n - # Skip lmms-eval parent task placeholders (no numeric metrics, just alias) + # Lmms-eval emits some groups (e.g. mvbench) as empty-placeholder + # parents whose actual values live on the children in + # `group_subtasks`. Aggregate (mean) to recover the headline. if set(task_results.keys()) <= {"alias", " ", ""}: - continue - - performance, metric_name = _resolve_metric( - task_name_clean, task_results, task_metrics - ) + subtasks = group_subtasks_map.get(task_name_clean, []) + if not subtasks: + continue + child_values: list[float] = [] + child_metric_name: str | None = None + for subtask_name in subtasks: + sub_results = results.get(subtask_name, {}) + sub_val, sub_metric = _resolve_metric( + task_name_clean, sub_results, task_metrics + ) + if sub_val is not None: + child_values.append(sub_val) + if child_metric_name is None: + child_metric_name = sub_metric + if not child_values: + continue + performance = sum(child_values) / len(child_values) + metric_name = child_metric_name + else: + performance, metric_name = _resolve_metric( + task_name_clean, task_results, task_metrics + ) if performance is not None: if check: @@ -337,6 +415,9 @@ def collect_results( "task": task_name_clean, "n_shot": n_shot, "performance": performance, + "performance_normalized": _normalize_to_100( + performance, metric_name + ), "metric_name": metric_name if metric_name is not None else "", } ) @@ -436,24 +517,17 @@ def collect_results( # Structured output: versioned JSON and Markdown report # --------------------------------------------------------------------------- -SCHEMA_VERSION = "1.0" +SCHEMA_VERSION = "1.1" def write_results_json( rows: list[dict], output_path: str | Path, ) -> None: - """Write evaluation results as a versioned JSON file. + """Write versioned JSON: {version, generated_at, results: [...]}. - The schema is:: - - { - "version": "1.0", - "generated_at": "2026-04-02T12:00:00+00:00", - "results": [ - {"model": ..., "task": ..., "n_shot": ..., "metric": ..., "performance": ...} - ] - } + Each result row has `performance` (raw engine value) and + `performance_normalized` (0-100 via METRIC_NATIVE_SCALE, or null). """ output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) @@ -467,6 +541,7 @@ def write_results_json( "n_shot": row.get("n_shot", 0), "metric": row.get("metric_name", ""), "performance": row.get("performance", 0.0), + "performance_normalized": row.get("performance_normalized"), } ) @@ -483,20 +558,40 @@ def write_results_markdown( rows: list[dict], output_path: str | Path, ) -> None: - """Write evaluation results as a Markdown table.""" + """Write a Markdown results table on a 0-100 normalized scale.""" output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) lines = [ - "| Model | Task | N-shot | Metric | Performance |", - "|-------|------|--------|--------|-------------|", + "| Model | Task | N-shot | Metric | Performance (0–100) |", + "|-------|------|--------|--------|---------------------|", ] + has_raw_fallback = False + has_lower_is_better = False for row in rows: model = row.get("model_name", "") task = row.get("task", "") n_shot = row.get("n_shot", 0) metric = row.get("metric_name", "") - perf = row.get("performance", 0.0) - lines.append(f"| {model} | {task} | {n_shot} | {metric} | {perf:.4f} |") + normalized = row.get("performance_normalized") + if normalized is not None: + perf_cell = f"{normalized:.2f}" + else: + raw = row.get("performance", 0.0) + perf_cell = f"{raw:.4f}*" + has_raw_fallback = True + if any(k in metric.lower() for k in ("wer", "mer", "cer")): + has_lower_is_better = True + lines.append(f"| {model} | {task} | {n_shot} | {metric} | {perf_cell} |") + + # Only emit footnotes that apply to this report. + footnotes = [] + if has_raw_fallback: + footnotes.append("> `*` = raw value (metric scale not in `METRIC_NATIVE_SCALE`).") + if has_lower_is_better: + footnotes.append("> WER/MER/CER are lower-is-better.") + if footnotes: + lines.append("") + lines.extend(footnotes) output_path.write_text("\n".join(lines) + "\n") diff --git a/oellm/scheduler.py b/oellm/scheduler.py index 61a80ddd..c298c4df 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -117,6 +117,7 @@ def schedule_evals( lm_eval_include_path: str | None = None, local: bool = False, slurm_template_var: str | None = None, + allow_missing_judge: bool = False, ) -> None: """ Schedule evaluation jobs for a given set of models, tasks, and number of shots. @@ -267,6 +268,16 @@ def schedule_evals( ] ) + # Refuse judge-required tasks without OPENAI_API_KEY (unless explicitly + # opted out). Runs before any model download / SLURM work so the user + # sees the failure immediately, not after a long pre-flight. + from oellm.utils import check_judge_llm_pre_flight + + check_judge_llm_pre_flight( + {job.task_path for job in eval_jobs}, + allow_missing=allow_missing_judge, + ) + expanded_eval_jobs = [] for job in eval_jobs: local_model_paths = _expand_local_model_paths(job.model_path) diff --git a/oellm/task_groups.py b/oellm/task_groups.py index d930532d..dd684ed9 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -1,5 +1,5 @@ from collections.abc import Iterable -from dataclasses import dataclass +from dataclasses import dataclass, field from importlib.resources import files import yaml @@ -10,6 +10,9 @@ class DatasetSpec: repo_id: str subset: str | None = None needs_snapshot_download: bool = False + # HF dataset revisions to pre-fetch. Most datasets only need `main`; + # OpenGVLab/MVBench keeps videos on a separate `video` branch. + revisions: list[str] = field(default_factory=lambda: ["main"]) @dataclass @@ -21,6 +24,7 @@ class _Task: hf_models: list[str] | None = None hf_dataset_files: list[dict] | None = None suite: str | None = None + revisions: list[str] | None = None @dataclass @@ -53,6 +57,7 @@ def from_dict(cls, name: str, data: dict) -> "TaskGroup": task_subset = task_data.get("subset") task_hf_models = task_data.get("hf_models") task_hf_dataset_files = task_data.get("hf_dataset_files") + task_revisions = task_data.get("revisions") tasks.append( _Task( name=task_name, @@ -62,6 +67,7 @@ def from_dict(cls, name: str, data: dict) -> "TaskGroup": hf_models=task_hf_models, hf_dataset_files=task_hf_dataset_files, suite=task_data.get("suite"), + revisions=task_revisions, ) ) @@ -198,37 +204,51 @@ def _extract_flores_subsets(task_name: str) -> list[str]: def _collect_dataset_specs(group_names: Iterable[str]) -> list[DatasetSpec]: parsed = _parse_task_groups([str(n).strip() for n in group_names if str(n).strip()]) - specs: list[DatasetSpec] = [] - seen: set[tuple[str, str | None, str | None]] = set() + # Merge specs sharing (repo_id, subset): union their revisions lists. + by_key: dict[tuple[str, str | None], DatasetSpec] = {} + order: list[tuple[str, str | None]] = [] def add_spec( dataset: str | None, subset: str | None, needs_snapshot_download: bool = False, + revisions: list[str] | None = None, ): if dataset is None: return + revs = list(revisions) if revisions else ["main"] key = (dataset, subset) - if key not in seen: - seen.add(key) - specs.append( - DatasetSpec( - repo_id=dataset, - subset=subset, - needs_snapshot_download=needs_snapshot_download, - ) + existing = by_key.get(key) + if existing is None: + by_key[key] = DatasetSpec( + repo_id=dataset, + subset=subset, + needs_snapshot_download=needs_snapshot_download, + revisions=revs, ) + order.append(key) + else: + for r in revs: + if r not in existing.revisions: + existing.revisions.append(r) + if needs_snapshot_download and not existing.needs_snapshot_download: + existing.needs_snapshot_download = True for t, _, group_name in _iter_all_tasks(parsed): needs_snapshot = group_name.startswith(("audio-", "video-")) if t.dataset == "facebook/flores" and not t.subset: for lang in _extract_flores_subsets(t.name): - add_spec(t.dataset, lang) + add_spec(t.dataset, lang, revisions=t.revisions) else: - add_spec(t.dataset, t.subset, needs_snapshot_download=needs_snapshot) + add_spec( + t.dataset, + t.subset, + needs_snapshot_download=needs_snapshot, + revisions=t.revisions, + ) - return specs + return [by_key[k] for k in order] def _collect_hf_model_repos(group_names: Iterable[str]) -> list[str]: @@ -291,13 +311,16 @@ def _build_task_dataset_map() -> dict[str, list[DatasetSpec]]: for t, _, _gname in _iter_all_tasks(parsed): if t.dataset and t.name not in task_map: + revs = list(t.revisions) if t.revisions else ["main"] if t.dataset == "facebook/flores" and not t.subset: task_map[t.name] = [ - DatasetSpec(repo_id=t.dataset, subset=lang) + DatasetSpec(repo_id=t.dataset, subset=lang, revisions=list(revs)) for lang in _extract_flores_subsets(t.name) ] else: - task_map[t.name] = [DatasetSpec(repo_id=t.dataset, subset=t.subset)] + task_map[t.name] = [ + DatasetSpec(repo_id=t.dataset, subset=t.subset, revisions=revs) + ] return task_map diff --git a/oellm/utils.py b/oellm/utils.py index 16a53b9f..eb00e77e 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -379,9 +379,44 @@ def _pre_download_hf_dataset_files(dataset_files: list[dict]) -> None: logging.warning(f"Failed to download dataset files from '{repo_id}': {e}") +def _materialize_external_urls(ds, *, max_workers: int = 16) -> None: + """Iterate every row to force HF ``dl_manager`` to fetch external URLs. + + Datasets like ``facebook/textvqa`` store image URLs (not bytes) in + parquet rows; only per-row access triggers the HTTP fetch into the + cache. Strict: exceptions propagate so ``_pre_download_datasets_…`` + aborts the schedule before SLURM submission. + """ + if ds is None: + return + + from concurrent.futures import ThreadPoolExecutor + + def _materialize_split(split) -> None: + n = len(split) + if n == 0: + return + with ThreadPoolExecutor(max_workers=max_workers) as pool: + for _ in pool.map(lambda i: split[i], range(n)): + pass + + if hasattr(ds, "keys"): + for split_name in list(ds.keys()): + _materialize_split(ds[split_name]) + elif hasattr(ds, "__len__") and hasattr(ds, "__getitem__"): + # Skip anything that isn't a recognizable dataset shape (e.g. test stubs). + _materialize_split(ds) + + def _pre_download_datasets_from_specs( specs: Iterable, trust_remote_code: bool = True ) -> None: + """Pre-fetch every dataset spec into the local HF cache. + + Strict: any failure raises ``RuntimeError`` and aborts the schedule + before SLURM submission — compute nodes run ``HF_HUB_OFFLINE=1`` and + can't recover from a cache miss. Override with ``--skip-checks``. + """ from datasets import get_dataset_config_names, load_dataset from huggingface_hub import snapshot_download @@ -390,6 +425,7 @@ def _pre_download_datasets_from_specs( return console = get_console() + failures: list[tuple[str, Exception]] = [] with console.status( f"Downloading datasets… {len(specs_list)} datasets", @@ -399,44 +435,60 @@ def _pre_download_datasets_from_specs( label = f"{spec.repo_id}" + (f"/{spec.subset}" if spec.subset else "") status.update(f"Downloading '{label}' ({idx}/{len(specs_list)})") + snapshot_failed = False + revisions = getattr(spec, "revisions", None) or ["main"] if spec.needs_snapshot_download: - try: - # max_workers=2 keeps concurrent HEAD requests below HF's - # per-IP rate limit for many-file audio/video repos (e.g. - # lmms-lab/WenetSpeech). Higher values trigger HTTP 429 - # and long exponential backoffs even with auth. - snapshot_download( - repo_id=spec.repo_id, - repo_type="dataset", - max_workers=2, - ) - except Exception as e: - logging.warning(f"Failed to snapshot_download '{spec.repo_id}': {e}") + # Iterate every requested revision (e.g. OpenGVLab/MVBench + # splits content across `main` and `video` branches). + for rev in revisions: + rev_label = f"{label}@{rev}" if rev != "main" else label + status.update(f"Downloading '{rev_label}' ({idx}/{len(specs_list)})") + try: + # max_workers=2 keeps HEAD requests under HF's per-IP + # rate limit; higher triggers HTTP 429. + snapshot_download( + repo_id=spec.repo_id, + repo_type="dataset", + revision=rev, + max_workers=2, + ) + except Exception as e: + snapshot_failed = True + logging.warning( + f"snapshot_download failed for '{rev_label}': {e}; " + f"falling back to load_dataset." + ) try: - load_dataset( + ds = load_dataset( spec.repo_id, name=spec.subset, trust_remote_code=trust_remote_code, ) + _materialize_external_urls(ds) except ValueError as e: if "Config name is missing" in str(e) and spec.subset is None: - configs = get_dataset_config_names( - spec.repo_id, trust_remote_code=trust_remote_code - ) - logging.info( - f"Dataset '{spec.repo_id}' requires config. " - f"Downloading all {len(configs)} configs." - ) - for cfg in configs: - status.update( - f"Downloading '{spec.repo_id}/{cfg}' ({idx}/{len(specs_list)})" + try: + configs = get_dataset_config_names( + spec.repo_id, trust_remote_code=trust_remote_code ) - load_dataset( - spec.repo_id, - name=cfg, - trust_remote_code=trust_remote_code, + logging.info( + f"Dataset '{spec.repo_id}' requires config. " + f"Downloading all {len(configs)} configs." ) + for cfg in configs: + status.update( + f"Downloading '{spec.repo_id}/{cfg}' " + f"({idx}/{len(specs_list)})" + ) + ds_cfg = load_dataset( + spec.repo_id, + name=cfg, + trust_remote_code=trust_remote_code, + ) + _materialize_external_urls(ds_cfg) + except Exception as inner: + failures.append((label, inner)) continue if "Feature type" in str(e) and "not found" in str(e): hf_datasets_cache = os.environ.get( @@ -450,77 +502,115 @@ def _pre_download_datasets_from_specs( f"datasets version ('{e}'). Delete the stale cache and re-run:\n\n" f" rm -rf {cache_dir}\n" ) from None - raise + failures.append((label, e)) + except Exception as e: + # Network / hub / OS errors — catch and aggregate, don't swallow. + failures.append((label, e)) + else: + logging.debug(f"Finished downloading dataset '{label}'.") + continue + + if snapshot_failed: + logging.debug( + f"Both snapshot_download and load_dataset failed for '{label}'." + ) + + if failures: + details = "\n".join( + f" - {label}: {type(e).__name__}: {e}" for label, e in failures + ) + raise RuntimeError( + f"Pre-download failed for {len(failures)}/{len(specs_list)} dataset(s); " + f"aborting before SLURM submission (compute nodes run offline).\n\n" + f"Failures:\n{details}\n\n" + f"Common fixes: set HF_TOKEN / accept dataset license on HF / retry " + f"after rate-limit cools off. Bypass with `--skip-checks` if the cache " + f"is already populated out-of-band.\n" + ) + + +_PACKAGE_ROOT = Path(__file__).resolve().parent +# Saved originals when capture is active. Module-level (not a closure) so +# the filtered_* functions are importable as `oellm.utils.filtered_*` — +# required for HF datasets' multiprocessing workers to resolve them when +# the patched print/logger gets pickled across processes. +_capture_originals: dict = {"active": False} - logging.debug(f"Finished downloading dataset '{label}'.") + +def _is_internal_stack(skip: int = 2, max_depth: int = 20) -> bool: + f = sys._getframe(skip) + depth = 0 + while f and depth < max_depth: + code = f.f_code + filename = code.co_filename if code else "" + if filename: + p = Path(filename).resolve() + name = code.co_name if code else "" + # Skip logging internals and our filtering wrappers to find the real caller. + if "/logging/__init__.py" in filename or name.startswith("filtered_"): + f = f.f_back + depth += 1 + continue + return p.is_relative_to(_PACKAGE_ROOT) + f = f.f_back + depth += 1 + return False + + +def filtered_print(*args, **kwargs): + orig = _capture_originals.get("print") + if orig is None or _is_internal_stack(): + return (orig or builtins.print)(*args, **kwargs) + return None + + +def filtered_logger_info(self, msg, *args, **kwargs): + orig = _capture_originals.get("logger_info") + if orig is None or _is_internal_stack(): + return (orig or logging.Logger.info)(self, msg, *args, **kwargs) + return None + + +def filtered_logger_debug(self, msg, *args, **kwargs): + orig = _capture_originals.get("logger_debug") + if orig is None or _is_internal_stack(): + return (orig or logging.Logger.debug)(self, msg, *args, **kwargs) + return None + + +def filtered_module_info(msg, *args, **kwargs): + orig = _capture_originals.get("module_info") + if orig is None or _is_internal_stack(): + return (orig or logging.info)(msg, *args, **kwargs) + return None + + +def filtered_module_debug(msg, *args, **kwargs): + orig = _capture_originals.get("module_debug") + if orig is None or _is_internal_stack(): + return (orig or logging.debug)(msg, *args, **kwargs) + return None @contextmanager def capture_third_party_output(verbose: bool = False): - """ - Suppresses print/logging.info/logging.debug originating from non-project modules - unless verbose=True. - - A call is considered "third-party" if its immediate caller's file path is not - under the repository root (parent of the `oellm` package directory). - """ + """Suppress print/logging.info/logging.debug from non-project modules + unless verbose=True. A call is "third-party" if its caller's file path + is not under the `oellm` package directory.""" if verbose: yield return - package_root = Path(__file__).resolve().parent - - def is_internal_stack(skip: int = 2, max_depth: int = 20) -> bool: - f = sys._getframe(skip) - depth = 0 - while f and depth < max_depth: - code = f.f_code - filename = code.co_filename if code else "" - if filename: - p = Path(filename).resolve() - name = code.co_name if code else "" - # Skip logging internals and our filtering wrappers to find the real caller - if "/logging/__init__.py" in filename or name.startswith("filtered_"): - f = f.f_back - depth += 1 - continue - return p.is_relative_to(package_root) - f = f.f_back - depth += 1 - return False - - orig_print = builtins.print - orig_logger_info = logging.Logger.info - orig_logger_debug = logging.Logger.debug - orig_module_info = logging.info - orig_module_debug = logging.debug - - def filtered_print(*args, **kwargs): - if is_internal_stack(): - return orig_print(*args, **kwargs) - # third-party: drop - return None - - def filtered_logger_info(self, msg, *args, **kwargs): - if is_internal_stack(): - return orig_logger_info(self, msg, *args, **kwargs) - return None - - def filtered_logger_debug(self, msg, *args, **kwargs): - if is_internal_stack(): - return orig_logger_debug(self, msg, *args, **kwargs) - return None - - def filtered_module_info(msg, *args, **kwargs): - if is_internal_stack(): - return orig_module_info(msg, *args, **kwargs) - return None - - def filtered_module_debug(msg, *args, **kwargs): - if is_internal_stack(): - return orig_module_debug(msg, *args, **kwargs) - return None - + _capture_originals.update( + { + "active": True, + "print": builtins.print, + "logger_info": logging.Logger.info, + "logger_debug": logging.Logger.debug, + "module_info": logging.info, + "module_debug": logging.debug, + } + ) builtins.print = filtered_print # type: ignore logging.Logger.info = filtered_logger_info # type: ignore[assignment] logging.Logger.debug = filtered_logger_debug # type: ignore[assignment] @@ -530,11 +620,12 @@ def filtered_module_debug(msg, *args, **kwargs): try: yield finally: - builtins.print = orig_print - logging.Logger.info = orig_logger_info # type: ignore[assignment] - logging.Logger.debug = orig_logger_debug # type: ignore[assignment] - logging.info = orig_module_info # type: ignore[assignment] - logging.debug = orig_module_debug # type: ignore[assignment] + builtins.print = _capture_originals["print"] + logging.Logger.info = _capture_originals["logger_info"] # type: ignore[assignment] + logging.Logger.debug = _capture_originals["logger_debug"] # type: ignore[assignment] + logging.info = _capture_originals["module_info"] # type: ignore[assignment] + logging.debug = _capture_originals["module_debug"] # type: ignore[assignment] + _capture_originals["active"] = False def capture_third_party_output_from_kwarg( @@ -565,3 +656,42 @@ def _filter_warnings(): warnings.filterwarnings("ignore", module="lm_eval") warnings.filterwarnings("ignore", module="lighteval") + + +def check_judge_llm_pre_flight( + tasks: Iterable[str], *, allow_missing: bool = False +) -> None: + """Refuse to schedule judge-graded tasks without ``OPENAI_API_KEY``. + + Runs before SLURM submission. Inspects ``tasks`` against + ``JUDGE_REQUIRED_TASKS`` and raises ``SystemExit`` if any are present + and ``OPENAI_API_KEY`` is unset, unless ``allow_missing=True`` (the + user explicitly opted in to letting those tasks emit null scores). + """ + import os + + from oellm.constants import JUDGE_REQUIRED_TASKS + + needed = sorted({t for t in tasks if t in JUDGE_REQUIRED_TASKS}) + if not needed: + return + if os.environ.get("OPENAI_API_KEY"): + return + if allow_missing: + logging.warning( + "Scheduling %d judge-required task(s) without OPENAI_API_KEY: %s. " + "These will emit null performance values in collect-results.", + len(needed), + ", ".join(needed), + ) + return + raise SystemExit( + "Refusing to schedule judge-required task(s) without OPENAI_API_KEY:\n" + f" {', '.join(needed)}\n\n" + "These tasks need an LLM judge / extractor to produce a valid metric. " + "Either:\n" + " - export OPENAI_API_KEY=... before re-running, or\n" + " - pass --allow-missing-judge to acknowledge that these tasks will " + "emit null scores, or\n" + " - remove them from the task list." + ) diff --git a/pyproject.toml b/pyproject.toml index 8dcdb8f0..31967faf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,15 +20,33 @@ dev = [ "pytest-timeout>=2.3.1", "pre-commit", ] +# Text-evaluation engine base (lm-eval-harness + dependencies). Pair with +# ``image`` / ``audio`` for a "general" venv that runs text + multimodal +# tasks side-by-side. ``lighteval`` cannot share a venv with lm-eval +# (datasets <4 vs >=4 conflict) and is installed separately via +# ``uv tool install`` — see docs/VENV.md. +text = [ + "lm-eval", + "torch", + "transformers", + "accelerate", +] +# NOTE: lmms-eval is NOT a pip dep — its wheel drops `_default_template_yaml` +# files. Install it editable (`uv pip install -e "git+…"`); see docs/VENV.md. image = [ - "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", + # transformers 4.50+ fast llava_onevision processor crashes on images=[]. + "transformers>=4.45,<4.50", + # Phi-4-multimodal trust_remote_code modeling imports backoff. + "backoff", ] video = [ - "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", + "transformers>=4.45,<4.50", + "backoff", + # PyAV 14 renamed AVError to OSError; torchvision still uses the old name. + "av<14", ] # HPC Singularity image must also include ffmpeg for non-WAV decode. audio = [ - "lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git", "soundfile", "librosa", "jiwer", @@ -46,6 +64,34 @@ audiobench = [ "soundfile", "librosa", ] +# Evalchemy reasoning suite — uses a forked lm-eval that cannot coexist +# with mainline lm-eval (different fork at the same import path). Install +# in its own venv: ``uv pip install '.[evalchemy]'``. Pinned versions are +# what evalchemy upstream requires; see docs/VENV.md. +evalchemy = [ + "lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@etashg/tokenize_fix", + "scipy==1.17.0", + "datasets==3.6.0", + "transformers==4.57.6", + "accelerate==1.12.0", + "bespokelabs-curator==0.1.26", + "sqlalchemy", + "torch", + "langdetect", + "immutabledict", +] +# DCLM-core-22 — pins ``lm-eval==0.4.9.2`` because v0.4.10+ breaks +# ``agieval_lsat_ar`` few-shot. Install in its own venv: +# ``uv pip install '.[dclm]'``. +dclm = [ + "lm-eval==0.4.9.2", + "torch", + "transformers>=4.43.2,<5.0.0", + "accelerate", + "wandb", + "sentencepiece", + "tiktoken", +] [project.scripts] oellm = "oellm.main:main" @@ -59,6 +105,26 @@ module-name = "oellm" module-root = "" include = ["oellm/resources/*", "oellm/resources/**/*"] +# Conflict groups: extras with incompatible pins (must live in separate +# venvs). text/image/video can coexist; evalchemy/dclm cannot share with +# anything pinning a different lm-eval or transformers version. See docs/VENV.md. +[tool.uv] +conflicts = [ + [ + { extra = "text" }, + { extra = "evalchemy" }, + { extra = "dclm" }, + ], + [ + { extra = "image" }, + { extra = "evalchemy" }, + ], + [ + { extra = "video" }, + { extra = "evalchemy" }, + ], +] + [tool.ruff] line-length = 90 target-version = "py312" diff --git a/requirements-venv-dclm.txt b/requirements-venv-dclm.txt deleted file mode 100644 index ef41986d..00000000 --- a/requirements-venv-dclm.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Dependencies for DCLM-core-22 evaluation (install in venv) -# Install with: uv pip install -r requirements-venv-dclm.txt -lm-eval==0.4.9.2 -torch -transformers>=4.43.2,<5.0.0 -accelerate -datasets<4.0.0 -wandb -sentencepiece -tiktoken diff --git a/requirements-venv-evalchemy.txt b/requirements-venv-evalchemy.txt deleted file mode 100644 index c63c3fbc..00000000 --- a/requirements-venv-evalchemy.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Dependencies for evalchemy evaluation - -# lm-eval fork used by evalchemy -lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@etashg/tokenize_fix - -scipy==1.17.0 -datasets==3.6.0 -transformers==4.57.6 -accelerate==1.12.0 -bespokelabs-curator==0.1.26 -sqlalchemy -torch - -langdetect -immutabledict diff --git a/requirements-venv.txt b/requirements-venv.txt deleted file mode 100644 index 433e3ed4..00000000 --- a/requirements-venv.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Dependencies for lm-eval and lmms-eval (install in venv) -# Install with: uv pip install -r requirements-venv.txt -lm-eval -torch -transformers -accelerate -datasets<4.0.0 - -# lmms-eval: image/video/audio evaluation engine (required for image-vqa task group) -# lmms-eval is compatible with datasets<4.0.0; install alongside lm-eval. -lmms-eval @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git - -# HPC Singularity image must also include ffmpeg for non-WAV decode. -soundfile -librosa -jiwer - -# lighteval must be installed separately as a uv tool to avoid datasets version conflict: -# UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ -# uv tool install --python 3.12 --with "langcodes[data]" --with "pillow" \ -# "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 7d6640f5..90b790fc 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -198,6 +198,53 @@ def test_mathvista_leaf_llm_judge_metric(self, tmp_path): df = run_collect(tmp_path, data) assert df.iloc[0]["performance"] == pytest.approx(0.49) + def test_mvbench_group_aggregates_from_subtasks(self, tmp_path): + """lmms-eval emits MVBench as an empty-placeholder parent + (``{" ": " ", "alias": "mvbench"}``) plus 20 metric-bearing + children, with the parent listed in ``group_subtasks``. Without an + aggregation fallback, both parent (empty placeholder) and children + (group-subtask skip) get dropped from the output. This test pins + the synthetic-row behavior — average of children's metrics.""" + data = { + "model_name": "qwen2_vl", + "model_name_or_path": "/models/Qwen2-VL-2B-Instruct", + "results": { + # Empty parent — what lmms-eval actually writes for MVBench. + "mvbench": {" ": " ", "alias": "mvbench"}, + "mvbench_action_antonym": { + "mvbench_action_antonym/mvbench_accuracy,none": 75.0, + }, + "mvbench_moving_direction": { + "mvbench_moving_direction/mvbench_accuracy,none": 100.0, + }, + "mvbench_action_sequence": { + "mvbench_action_sequence/mvbench_accuracy,none": 0.0, + }, + "mvbench_scene_transition": { + "mvbench_scene_transition/mvbench_accuracy,none": 25.0, + }, + }, + "n-shot": {"mvbench": 0}, + "group_subtasks": { + "mvbench": [ + "mvbench_action_antonym", + "mvbench_moving_direction", + "mvbench_action_sequence", + "mvbench_scene_transition", + ] + }, + } + df = run_collect(tmp_path, data) + + # One row for the mvbench group, none for its children. + assert len(df) == 1 + row = df.iloc[0] + assert row["task"] == "mvbench" + # Mean of the 4 children: (75 + 100 + 0 + 25) / 4 = 50. + assert row["performance"] == pytest.approx(50.0) + # The metric_name should come from a child (mvbench_accuracy,none). + assert "mvbench_accuracy" in row["metric_name"] + def test_multiple_image_tasks_in_one_file(self, tmp_path): data = { "model_name": "llava_hf", @@ -391,9 +438,14 @@ def test_json_file_written_alongside_csv(self, tmp_path): json_path = tmp_path / "out.json" assert json_path.exists() envelope = json.loads(json_path.read_text()) - assert envelope["version"] == "1.0" + assert envelope["version"] == "1.1" assert len(envelope["results"]) == 1 - assert envelope["results"][0]["task"] == "copa" + record = envelope["results"][0] + assert record["task"] == "copa" + # acc is a 0-1 scale metric in METRIC_NATIVE_SCALE, so the + # normalized value is the raw value × 100. + assert record["performance"] == pytest.approx(0.80) + assert record["performance_normalized"] == pytest.approx(80.0) def test_markdown_file_written_alongside_csv(self, tmp_path): results_dir = tmp_path / "results" diff --git a/tests/test_reporter.py b/tests/test_reporter.py index 57a21769..0b0de8d2 100644 --- a/tests/test_reporter.py +++ b/tests/test_reporter.py @@ -32,7 +32,7 @@ def test_write_json_schema_version(tmp_path: Path) -> None: out = tmp_path / "results.json" write_results_json(_SAMPLE_ROWS, out) data = json.loads(out.read_text()) - assert data["version"] == SCHEMA_VERSION == "1.0" + assert data["version"] == SCHEMA_VERSION == "1.1" def test_write_json_result_fields(tmp_path: Path) -> None: @@ -41,7 +41,14 @@ def test_write_json_result_fields(tmp_path: Path) -> None: data = json.loads(out.read_text()) assert len(data["results"]) == 2 for r in data["results"]: - assert set(r.keys()) == {"model", "task", "n_shot", "metric", "performance"} + assert set(r.keys()) == { + "model", + "task", + "n_shot", + "metric", + "performance", + "performance_normalized", + } def test_write_json_result_values(tmp_path: Path) -> None: @@ -70,7 +77,7 @@ def test_write_json_empty_rows(tmp_path: Path) -> None: write_results_json([], out) data = json.loads(out.read_text()) assert data["results"] == [] - assert data["version"] == "1.0" + assert data["version"] == "1.1" def test_write_json_creates_parent_dirs(tmp_path: Path) -> None: @@ -92,18 +99,50 @@ def test_write_markdown_header(tmp_path: Path) -> None: out = tmp_path / "results.md" write_results_markdown(_SAMPLE_ROWS, out) text = out.read_text() - assert "| Model | Task | N-shot | Metric | Performance |" in text - assert "|-------|------|--------|--------|-------------|" in text + assert "| Model | Task | N-shot | Metric | Performance (0–100) |" in text + assert "|-------|------|--------|--------|---------------------|" in text def test_write_markdown_data_row(tmp_path: Path) -> None: + """When ``performance_normalized`` is missing on the row dict (legacy + callers), the Markdown falls back to the raw value with a ``*`` suffix + so the reader knows the scale is not registered.""" out = tmp_path / "results.md" write_results_markdown(_SAMPLE_ROWS, out) text = out.read_text() - assert "0.7500" in text + assert "0.7500*" in text assert "vqav2" in text +def test_write_markdown_uses_normalized_when_present(tmp_path: Path) -> None: + out = tmp_path / "results.md" + rows = [ + { + "model_name": "/models/llava", + "task": "vqav2", + "n_shot": 0, + "performance": 0.755, + "performance_normalized": 75.5, + "metric_name": "exact_match", + }, + { + "model_name": "/models/llava", + "task": "mvbench", + "n_shot": 0, + "performance": 56.2, + "performance_normalized": 56.2, + "metric_name": "mvbench_accuracy", + }, + ] + write_results_markdown(rows, out) + text = out.read_text() + assert "| 75.50 |" in text + assert "| 56.20 |" in text + # No `*` suffix when normalized value is present. + assert "75.50*" not in text + assert "56.20*" not in text + + def test_write_markdown_creates_parent_dirs(tmp_path: Path) -> None: out = tmp_path / "reports" / "results.md" write_results_markdown(_SAMPLE_ROWS, out) diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index 325d1c82..083f12bd 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -22,6 +22,9 @@ def test_schedule_evals(tmp_path, n_shot, task_groups): patch("oellm.runner.detect_lmms_model_type", return_value="llava"), patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), ): + # ``allow_missing_judge`` so this parametrized smoke test can exercise + # judge-required task groups (audio-alpaca-audio, video-activitynet-qa, + # …) without setting ``OPENAI_API_KEY`` in CI. schedule_evals( models="EleutherAI/pythia-70m", task_groups=task_groups, @@ -29,6 +32,7 @@ def test_schedule_evals(tmp_path, n_shot, task_groups): skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, + allow_missing_judge=True, ) diff --git a/tests/test_utils.py b/tests/test_utils.py index ebc0163e..28fda323 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,21 @@ -from oellm.utils import _expand_local_model_paths, _num_jobs_in_queue +from dataclasses import dataclass, field + +import pytest + +from oellm.utils import ( + _expand_local_model_paths, + _materialize_external_urls, + _num_jobs_in_queue, + _pre_download_datasets_from_specs, +) + + +@dataclass +class _FakeSpec: + repo_id: str + subset: str | None = None + needs_snapshot_download: bool = False + revisions: list[str] = field(default_factory=lambda: ["main"]) class TestExpandLocalModelPaths: @@ -54,3 +71,377 @@ class Result: monkeypatch.setattr("oellm.utils.subprocess.run", lambda *a, **kw: Result()) assert _num_jobs_in_queue() == 0 + + +class TestPreDownloadFailsLoudly: + """Regression tests for the fail-loudly behavior in + `_pre_download_datasets_from_specs`. + + Compute nodes run with HF_HUB_OFFLINE=1, so a silent download failure on + the login node translates to a mid-job ConnectionError on the compute + node. The pre-download step must therefore *raise* on any unrecoverable + failure, with a clear message naming every failing dataset. + """ + + def test_raises_on_connection_error_from_load_dataset(self, monkeypatch): + """A ConnectionError from load_dataset must abort the schedule.""" + + def boom_load_dataset(*args, **kwargs): + raise ConnectionError("simulated offline / DNS failure") + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("datasets.load_dataset", boom_load_dataset) + + specs = [_FakeSpec(repo_id="some-org/some-dataset")] + + with pytest.raises(RuntimeError) as excinfo: + _pre_download_datasets_from_specs(specs) + + msg = str(excinfo.value) + # The error message must identify which dataset failed and what kind + # of error it was — otherwise the user has no way to debug. + assert "some-org/some-dataset" in msg + assert "ConnectionError" in msg + assert "1/1 dataset" in msg or "1 dataset" in msg + + def test_aggregates_multiple_failures(self, monkeypatch): + """All failures are reported in one RuntimeError, not just the first.""" + + def boom_load_dataset(repo_id, **kwargs): + raise OSError(f"network unavailable for {repo_id}") + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("datasets.load_dataset", boom_load_dataset) + + specs = [ + _FakeSpec(repo_id="org/dataset-a"), + _FakeSpec(repo_id="org/dataset-b"), + _FakeSpec(repo_id="org/dataset-c"), + ] + + with pytest.raises(RuntimeError) as excinfo: + _pre_download_datasets_from_specs(specs) + + msg = str(excinfo.value) + assert "org/dataset-a" in msg + assert "org/dataset-b" in msg + assert "org/dataset-c" in msg + assert "3/3" in msg or "3 dataset" in msg + + def test_passes_silently_when_all_specs_succeed(self, monkeypatch): + """The happy path remains silent and non-raising.""" + calls = [] + + def fake_load_dataset(repo_id, **kwargs): + calls.append(repo_id) + return object() + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) + + specs = [ + _FakeSpec(repo_id="org/dataset-a"), + _FakeSpec(repo_id="org/dataset-b"), + ] + + # No exception expected. + _pre_download_datasets_from_specs(specs) + assert calls == ["org/dataset-a", "org/dataset-b"] + + def test_snapshot_failure_alone_does_not_raise_if_load_dataset_works( + self, monkeypatch + ): + """snapshot_download is best-effort; load_dataset success is enough.""" + + def boom_snapshot(*args, **kwargs): + raise OSError("simulated snapshot HTTP 429") + + def fake_load_dataset(*args, **kwargs): + return object() + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("huggingface_hub.snapshot_download", boom_snapshot) + monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) + + specs = [_FakeSpec(repo_id="org/dataset-a", needs_snapshot_download=True)] + + # snapshot_download fails but load_dataset succeeds → no raise. + _pre_download_datasets_from_specs(specs) + + +class TestPreDownloadRevisions: + """Tests for DatasetSpec.revisions handling in pre-download. + + Datasets like OpenGVLab/MVBench split content across branches: parquet + metadata on `main`, video assets on `video`. snapshot_download must be + called once per revision; the default of ["main"] preserves the + legacy single-snapshot behavior for all other datasets. + """ + + def test_snapshot_download_called_once_per_revision(self, monkeypatch): + snapshot_calls = [] + + def fake_snapshot(*, repo_id, repo_type, revision, max_workers): + snapshot_calls.append((repo_id, revision)) + + def fake_load_dataset(*args, **kwargs): + return object() + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot) + monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) + + specs = [ + _FakeSpec( + repo_id="OpenGVLab/MVBench", + needs_snapshot_download=True, + revisions=["main", "video"], + ) + ] + + _pre_download_datasets_from_specs(specs) + + assert snapshot_calls == [ + ("OpenGVLab/MVBench", "main"), + ("OpenGVLab/MVBench", "video"), + ] + + def test_default_revisions_falls_back_to_main(self, monkeypatch): + """Specs without an explicit revisions list still snapshot 'main'.""" + snapshot_calls = [] + + def fake_snapshot(*, repo_id, repo_type, revision, max_workers): + snapshot_calls.append((repo_id, revision)) + + def fake_load_dataset(*args, **kwargs): + return object() + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot) + monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) + + # Spec with default revisions=["main"]. + specs = [_FakeSpec(repo_id="some-org/dataset", needs_snapshot_download=True)] + + _pre_download_datasets_from_specs(specs) + + assert snapshot_calls == [("some-org/dataset", "main")] + + +class TestMaterializeExternalUrls: + """`_materialize_external_urls` forces HF datasets' lazy URL fetches by + accessing EVERY row of EVERY split. + + Datasets like `facebook/textvqa` store image URLs as row fields rather + than embedded image bytes; the actual HTTP fetch is deferred until each + row is read. We must trigger it here on the login node so the cache is + complete before SLURM submission. Touching only the first row leaves + 99%+ of URLs un-cached and the compute-node job fails with + ConnectionError on the second sample. + """ + + def test_reads_every_row_for_each_split_of_dataset_dict(self): + """Every index of every split is accessed — that's what triggers + the per-row URL download for URL-typed Image columns.""" + import threading + from collections import defaultdict + + accessed = defaultdict(set) + lock = threading.Lock() + + class _FakeSplit: + def __init__(self, name, n): + self.name = name + self.n = n + + def __len__(self): + return self.n + + def __getitem__(self, idx): + with lock: + accessed[self.name].add(idx) + return {"image": b"", "answer": "x"} + + class _FakeDatasetDict: + def __init__(self): + self._splits = { + "train": _FakeSplit("train", 25), + "test": _FakeSplit("test", 10), + } + + def keys(self): + return self._splits.keys() + + def __getitem__(self, split): + return self._splits[split] + + _materialize_external_urls(_FakeDatasetDict()) + + assert accessed["train"] == set(range(25)) + assert accessed["test"] == set(range(10)) + + def test_reads_every_row_for_bare_dataset(self): + """For a bare Dataset (no .keys()), every index is accessed.""" + import threading + + accessed = set() + lock = threading.Lock() + + class _FakeDataset: + def __len__(self): + return 30 + + def __getitem__(self, idx): + with lock: + accessed.add(idx) + return {"image": b""} + + _materialize_external_urls(_FakeDataset()) + + assert accessed == set(range(30)) + + def test_skips_empty_splits(self): + """An empty dataset shouldn't cause IndexError.""" + + class _Empty: + def __len__(self): + return 0 + + def __getitem__(self, idx): # pragma: no cover — should not be called + raise AssertionError( + "Empty dataset must not be indexed by _materialize_external_urls" + ) + + _materialize_external_urls(_Empty()) + + def test_none_input_is_a_no_op(self): + _materialize_external_urls(None) + + def test_exception_during_materialization_propagates(self): + """Strict mode: any exception during materialization is propagated. + + The outer ``_pre_download_datasets_from_specs`` loop catches it and + records the failure. Silently swallowing would let an incomplete + cache proceed to SLURM submission and cause a compute-node + ConnectionError that's invisible until you read per-job stderr. + """ + + class _Brittle: + def __len__(self): + return 10 + + def __getitem__(self, idx): + raise ConnectionError("simulated external URL fetch failure") + + with pytest.raises(ConnectionError): + _materialize_external_urls(_Brittle(), max_workers=1) + + def test_materialize_failure_aggregated_into_pre_download_failures(self, monkeypatch): + """End-to-end: when materialization fails, the outer pre-download + function records the failure and raises a RuntimeError. The schedule + is aborted before any SLURM job is submitted.""" + + class _Brittle: + def __len__(self): + return 5 + + def __getitem__(self, idx): + raise ConnectionError("upstream URL unreachable") + + def fake_load_dataset(*args, **kwargs): + return _Brittle() + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) + + specs = [_FakeSpec(repo_id="some-org/url-only-dataset")] + + with pytest.raises(RuntimeError) as excinfo: + _pre_download_datasets_from_specs(specs) + + msg = str(excinfo.value) + assert "some-org/url-only-dataset" in msg + assert "ConnectionError" in msg + + +def _resolve_filtered_print_in_worker(out_queue): + """Worker target: import `oellm.utils` and look up `filtered_print` as + a module attribute. Mirrors what HF datasets' multiprocessing workers + do when they unpickle a reference to the patched `print`.""" + try: + from oellm import utils as _u # noqa: PLC0415 + + fn = _u.filtered_print + out_queue.put(("ok", fn.__name__)) + except Exception as e: + out_queue.put(("err", f"{type(e).__name__}: {e}")) + + +class TestFilteredFunctionsAreModuleAttributes: + """Regression guard: the filtered_* functions used by + `capture_third_party_output` must be true module-level attributes, + not closures. + + HF datasets' Audio feature decoder spawns multiprocessing workers + that pickle/unpickle references to the patched `print`. When + `filtered_print` was a local closure, workers raised + ``AttributeError: module 'oellm.utils' has no attribute + 'filtered_print'`` — which surfaced as the audio pre-download + failing during `_materialize_external_urls`. + """ + + def test_module_exports_filtered_functions(self): + """Each filtered_* function must be reachable via getattr on the + module — this is what pickle's resolution uses.""" + from oellm import utils + + for name in ( + "filtered_print", + "filtered_logger_info", + "filtered_logger_debug", + "filtered_module_info", + "filtered_module_debug", + ): + fn = getattr(utils, name, None) + assert callable(fn), f"oellm.utils.{name} must be a module attribute" + assert fn.__module__ == "oellm.utils", ( + f"{name}.__module__ is {fn.__module__!r}; " + f"expected 'oellm.utils'. Closures defined inside " + f"capture_third_party_output won't resolve in multiprocessing workers." + ) + + def test_filtered_print_resolvable_in_multiprocessing_worker(self): + """End-to-end: a child process can resolve and call filtered_print.""" + import multiprocessing as _mp + + ctx = _mp.get_context("spawn") + q = ctx.Queue() + p = ctx.Process(target=_resolve_filtered_print_in_worker, args=(q,)) + p.start() + p.join(timeout=30) + assert p.exitcode == 0, f"Worker process crashed; exitcode={p.exitcode}" + status, payload = q.get(timeout=5) + assert status == "ok", ( + f"Worker failed to resolve oellm.utils.filtered_print: {payload}. " + f"This means a closure regression has been re-introduced and " + f"HF datasets' multiprocessing workers will crash again." + ) + + +class _NoopConsole: + """Stub for rich.console.Console that satisfies the `.status()` + context-manager interface used by _pre_download_datasets_from_specs.""" + + def status(self, *args, **kwargs): + return _NoopStatus() + + +class _NoopStatus: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def update(self, *args, **kwargs): + pass diff --git a/tests/test_video_task_groups.py b/tests/test_video_task_groups.py index 21cf2e8c..9cc4a74b 100644 --- a/tests/test_video_task_groups.py +++ b/tests/test_video_task_groups.py @@ -18,7 +18,8 @@ "video_mmmu_perception", "video_mmmu_comprehension", "video_mmmu_adaptation", - "egoschema", + "mvbench", + "egoschema_subset", "videomme", "activitynetqa", "longvideobench_val_v", @@ -26,6 +27,7 @@ EXPECTED_DATASETS = { "lmms-lab/VideoMMMU", + "OpenGVLab/MVBench", "lmms-lab/egoschema", "lmms-lab/Video-MME", "lmms-lab/ActivityNetQA", @@ -43,16 +45,18 @@ def test_video_understanding_suite_is_lmms_eval(self): suite = data["task_groups"][VIDEO_TASK_GROUP]["suite"] assert suite == "lmms_eval" - def test_video_understanding_has_seven_tasks(self): + def test_video_understanding_has_eight_tasks(self): data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) tasks = data["task_groups"][VIDEO_TASK_GROUP]["tasks"] - # 3 video_mmmu_* leaves + egoschema + videomme + activitynetqa + longvideobench - assert len(tasks) == 7 + # 3 video_mmmu_* leaves + mvbench + egoschema_subset + videomme + # + activitynetqa + longvideobench + assert len(tasks) == 8 def test_individual_video_groups_present(self): all_groups = get_all_task_group_names() for name in [ "video-videommmu", + "video-mvbench", "video-egoschema", "video-videomme", "video-activitynet-qa", @@ -119,6 +123,32 @@ def test_needs_snapshot_download_flag_set_on_specs(self): f"DatasetSpec for {s.repo_id} missing needs_snapshot_download=True flag" ) + def test_mvbench_has_main_and_video_revisions(self): + """OpenGVLab/MVBench splits content across two branches: `main` + (parquet metadata) and `video` (.mp4 files). Both must be + pre-downloaded for offline compute-node evaluation. This is the + regression guard for the `revisions: [main, video]` YAML field.""" + specs = _collect_dataset_specs([VIDEO_TASK_GROUP]) + mvbench = next((s for s in specs if s.repo_id == "OpenGVLab/MVBench"), None) + assert mvbench is not None, "MVBench spec missing from video-understanding" + assert mvbench.revisions == ["main", "video"], ( + f"MVBench DatasetSpec.revisions is {mvbench.revisions!r}, " + f"expected ['main', 'video']. Check that the `revisions:` field is " + f"set on the mvbench task entry in task-groups.yaml." + ) + + def test_default_revisions_is_main_for_other_video_specs(self): + """Other video datasets that don't set `revisions:` in YAML must + still pre-download `main` by default.""" + specs = _collect_dataset_specs([VIDEO_TASK_GROUP]) + for s in specs: + if s.repo_id == "OpenGVLab/MVBench": + continue + assert s.revisions == ["main"], ( + f"DatasetSpec for {s.repo_id} has unexpected revisions " + f"{s.revisions!r}; default should be ['main']." + ) + class TestVideoTaskGroupScheduleEvals: """Verify video-understanding integrates with the schedule_evals dry-run path.""" @@ -141,6 +171,7 @@ def test_schedule_evals_dry_run_video(self, tmp_path): skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, + allow_missing_judge=True, ) sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) @@ -168,6 +199,7 @@ def test_schedule_evals_jobs_csv_has_lmms_eval_suite(self, tmp_path): skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, + allow_missing_judge=True, ) csv_files = list(tmp_path.glob("**/jobs.csv")) From 5a898fb18ed610a064a298ee5d73b995318ac707 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Sat, 6 Jun 2026 06:01:11 +0200 Subject: [PATCH 24/44] fixes --- README.md | 36 ++++++++++++------------ docs/LEONARDO.md | 4 +-- docs/TASKS.md | 4 +-- docs/VENV.md | 10 +++---- oellm/contrib/CONTRIBUTING.md | 2 +- oellm/contrib/README.md | 2 +- oellm/contrib/audiobench/README.md | 10 +++---- oellm/contrib/regiondial_bench/README.md | 12 ++++---- oellm/main.py | 10 +++---- oellm/resources/template.sbatch | 2 +- oellm/results.py | 2 +- oellm/scheduler.py | 4 +-- oellm/utils.py | 2 +- pyproject.toml | 4 +-- tests/integration/test_slurm.py | 16 +++++------ 15 files changed, 60 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 4c0cf57a..1ad10e07 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,12 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across uv tool install -p 3.12 git+https://github.com/elliot-project/elliot-cli.git # Run evaluations using a task group -oellm schedule-eval \ +oellm-eval schedule \ --models "EleutherAI/pythia-160m" \ --task-groups "open-sci-0.01" # Image evaluation (requires venv with lmms-eval) -oellm schedule-eval \ +oellm-eval schedule \ --models "llava-hf/llava-1.5-7b-hf" \ --task-groups "image-vqa" \ --venv-path ~/elliot-venv @@ -118,26 +118,26 @@ Community-contributed benchmarks that run outside the standard evaluation engine ```bash # Run all 8 image benchmarks -oellm schedule-eval \ +oellm-eval schedule \ --models "llava-hf/llava-1.5-7b-hf" \ --task-groups "image-vqa" \ --venv-path ~/elliot-venv # Run all 5 video benchmarks -oellm schedule-eval \ +oellm-eval schedule \ --models "lmms-lab/llava-onevision-7b" \ --task-groups "video-understanding" \ --venv-path ~/elliot-venv # Mix image and text benchmarks in one submission -oellm schedule-eval \ +oellm-eval schedule \ --models "llava-hf/llava-1.5-7b-hf" \ --task-groups "image-mmbench,open-sci-0.01" \ --venv-path ~/elliot-venv # Use multiple task groups or a super group -oellm schedule-eval --models "model-name" --task-groups "belebele-eu-5-shot,global-mmlu-eu" -oellm schedule-eval --models "model-name" --task-groups "oellm-multilingual" +oellm-eval schedule --models "model-name" --task-groups "belebele-eu-5-shot,global-mmlu-eu" +oellm-eval schedule --models "model-name" --task-groups "oellm-multilingual" ``` ## Running Locally (without SLURM) @@ -154,7 +154,7 @@ oellm-eval schedule \ --tasks "gsm8k" \ --n-shot 0 \ --venv-path .venv \ - --local true \ + --local \ --limit 1 ``` @@ -165,7 +165,7 @@ Results are written to `./oellm-output//results/`. ```bash export HF_HOME=/leonardo_work/OELLM_prod2026/users/shaldar0/oellm-evals/hf_data export HF_HUB_OFFLINE=1 -oellm-eval schedule ... --venv_path .venv --local true +oellm-eval schedule ... --venv-path .venv --local ``` The `HF_HUB_OFFLINE` value is read when you invoke `oellm-eval` and baked into the generated script. @@ -176,11 +176,11 @@ Override cluster defaults (partition, account, time limit, memory, etc.) with `- ```bash # Use a different partition (e.g. dev-g on LUMI when small-g is crowded) -oellm schedule-eval --models "model-name" --task-groups "open-sci-0.01" \ +oellm-eval schedule --models "model-name" --task-groups "open-sci-0.01" \ --slurm-template-var '{"PARTITION":"dev-g"}' # Multiple overrides: partition, account, time limit, GPUs, exact RAM -oellm schedule-eval --models "model-name" --task-groups "open-sci-0.01" \ +oellm-eval schedule --models "model-name" --task-groups "open-sci-0.01" \ --slurm-template-var '{"PARTITION":"dev-g","ACCOUNT":"myproject","TIME":"02:00:00","GPUS_PER_NODE":2,"SLURM_MEM":"96G"}' ``` @@ -206,7 +206,7 @@ If you need full manual control over all model args, set `MODEL_ARGS`, for example: ```bash -MODEL_ARGS='batch_size=8' oellm schedule-eval \ +MODEL_ARGS='batch_size=8' oellm-eval schedule \ --models "model-name" --task-groups "belebele-eu-cf" --venv-path .venv ``` @@ -219,7 +219,7 @@ If you use custom tasks via `--tasks` that are not in the task groups registry, **Recommendation:** Use `--task-groups` when possible, or ensure your custom task datasets are already cached in `$HF_HOME` before scheduling. ## Collecting Results -After evaluations complete, collect results into a CSV. `collect-results` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs: +After evaluations complete, collect results into a CSV. `collect` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs: ``` output/ @@ -236,10 +236,10 @@ output/ ```bash # Basic collection -oellm collect-results --results-dir /path/to/eval-output-dir +oellm-eval collect /path/to/eval-output-dir # Check for missing evaluations and create a CSV for re-running them -oellm collect-results --results-dir /path/to/eval-output-dir --check true --output-csv results.csv +oellm-eval collect /path/to/eval-output-dir --check --output-csv results.csv ``` All `jobs.csv` files found under `results_dir` are merged into one; if the same `(model_path, task_path, n_shot)` row appears in multiple files the later-sorted entry wins (override duplicates). The merged jobs list is then compared against all `.json` result files found recursively. @@ -247,7 +247,7 @@ All `jobs.csv` files found under `results_dir` are merged into one; if the same The `--check` flag outputs a `results_missing.csv` that can be used to re-schedule failed jobs: ```bash -oellm schedule-eval --eval-csv-path results_missing.csv +oellm-eval schedule --eval-csv-path results_missing.csv ``` ## CSV-Based Scheduling @@ -255,7 +255,7 @@ oellm schedule-eval --eval-csv-path results_missing.csv For full control, provide a CSV file with columns: `model_path`, `task_path`, `n_shot`, and optionally `eval_suite`: ```bash -oellm schedule-eval --eval-csv-path custom_evals.csv +oellm-eval schedule --eval-csv-path custom_evals.csv ``` ## Installation @@ -300,7 +300,7 @@ uv sync --extra dev uv run pytest tests/ -v # Download-only mode for testing -uv run oellm schedule-eval --models "EleutherAI/pythia-160m" --task-groups "open-sci-0.01" --download-only +uv run oellm-eval schedule --models "EleutherAI/pythia-160m" --task-groups "open-sci-0.01" --download-only ``` ## Documentation diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index 1f51de46..385b7f3e 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -149,12 +149,12 @@ export HF_HOME="$SCRATCH/hf_cache" ```zsh # Run evaluations using a task group (recommended) -oellm schedule-eval \ +oellm-eval schedule \ --models "microsoft/DialoGPT-medium,EleutherAI/pythia-160m" \ --task-groups "open-sci-0.01" # Or specify individual tasks -oellm schedule-eval \ +oellm-eval schedule \ --models "EleutherAI/pythia-160m" \ --tasks "hellaswag,mmlu" \ --n-shot 5 diff --git a/docs/TASKS.md b/docs/TASKS.md index d6ee1bc6..dc0a574e 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -49,7 +49,7 @@ task_groups: 2. Use it: ```bash -oellm schedule-eval --models "model-name" --task-groups "my-benchmark" +oellm-eval schedule --models "model-name" --task-groups "my-benchmark" ``` ## Adding an Image Task Group @@ -72,7 +72,7 @@ task_groups: Run with: ```bash -oellm schedule-eval --models "path/to/vlm" --task-groups "my-image-benchmark" +oellm-eval schedule --models "path/to/vlm" --task-groups "my-image-benchmark" ``` The lmms-eval model adapter (e.g. `llava_hf`, `qwen2_vl`) is auto-detected diff --git a/docs/VENV.md b/docs/VENV.md index f2174689..8748131d 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -57,13 +57,13 @@ Verify with: ```bash # Text evaluation -oellm schedule-eval \ +oellm-eval schedule \ --models HuggingFaceTB/SmolLM2-135M-Instruct \ --task-groups open-sci-0.01 \ --venv-path /path/to/.venv # Image evaluation (lmms-eval) -oellm schedule-eval \ +oellm-eval schedule \ --models path/to/vlm \ --task-groups image-vqa \ --venv-path /path/to/.venv @@ -98,7 +98,7 @@ oellm-eval schedule \ --models Qwen/Qwen3-0.6B-Base \ --task-groups dclm-core-22 \ --venv-path dclm-core-venv \ - --skip-checks true + --skip-checks ``` ## Evalchemy (reasoning) @@ -124,11 +124,11 @@ We use [Ali's fork](https://github.com/Ali-Elganzory/evalchemy) which includes a 3. Run with `EVALCHEMY_DIR` pointing to the cloned repo: ```bash export HF_ALLOW_CODE_EVAL=1 # required by MBPP - EVALCHEMY_DIR=$(pwd)/evalchemy oellm schedule-eval \ + EVALCHEMY_DIR=$(pwd)/evalchemy oellm-eval schedule \ --models HuggingFaceTB/SmolLM2-135M \ --task-groups reasoning \ --venv-path evalchemy-venv \ - --skip-checks true + --skip-checks ``` > **Note:** `HF_ALLOW_CODE_EVAL=1` is required because MBPP (run via lm-eval-harness) uses HuggingFace's `code_eval` metric which executes model-generated code. The evalchemy benchmarks (GPQADiamond, MATH500, LiveCodeBench) do not require this variable as they handle code execution safely through internal guards. diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index f1a5992d..9010aec6 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -23,7 +23,7 @@ task_groups: ``` ```bash -oellm schedule-eval \ +oellm-eval schedule \ --models org/MyModel \ --task-groups my-benchmark \ --venv-path ~/elliot-venv diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md index bd08efa0..917d8f23 100644 --- a/oellm/contrib/README.md +++ b/oellm/contrib/README.md @@ -15,7 +15,7 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). **Metrics:** gIoU (primary), cIoU, bbox_AP, pass_rate@0.3/0.5/0.7/0.9, per-round R1–R7 ```bash -oellm schedule-eval \ +oellm-eval schedule \ --models lmsdss/RegionReasoner-7B \ --task-groups regiondial-bench \ --venv-path ~/elliot-venv diff --git a/oellm/contrib/audiobench/README.md b/oellm/contrib/audiobench/README.md index 60ab3261..d51590b8 100644 --- a/oellm/contrib/audiobench/README.md +++ b/oellm/contrib/audiobench/README.md @@ -96,7 +96,7 @@ uv pip install --reinstall rapidfuzz ### Dataset pre-download -No manual steps required. `schedule-eval` pre-downloads every +No manual steps required. `schedule` pre-downloads every `AudioLLMs/*` HF repo referenced by the requested task group on the login node via `huggingface_hub.snapshot_download(max_workers=2)`, so compute nodes do not need internet access. @@ -116,19 +116,19 @@ nodes do not need internet access. ```bash # Full AudioBench suite on a Qwen2-Audio model: -oellm schedule-eval \ +oellm-eval schedule \ --models Qwen/Qwen2-Audio-7B-Instruct \ --task-groups audio-audiobench \ --venv-path audiobench-venv # ASR only: -oellm schedule-eval \ +oellm-eval schedule \ --models Qwen/Qwen2-Audio-7B-Instruct \ --task-groups audio-audiobench-asr \ --venv-path audiobench-venv # Smoke test with --limit: -oellm schedule-eval \ +oellm-eval schedule \ --models Qwen/Qwen2-Audio-7B-Instruct \ --task-groups audio-audiobench-asr \ --limit 100 \ @@ -141,7 +141,7 @@ unset, the full test split is evaluated. ### Collecting results ```bash -oellm collect-results \ +oellm-eval collect \ --eval-output-dir /path/to/evals \ --output-csv audiobench_results.csv ``` diff --git a/oellm/contrib/regiondial_bench/README.md b/oellm/contrib/regiondial_bench/README.md index 57f646ea..fea13252 100644 --- a/oellm/contrib/regiondial_bench/README.md +++ b/oellm/contrib/regiondial_bench/README.md @@ -71,7 +71,7 @@ uv pip install pi-heif ### What gets auto-downloaded -`oellm schedule-eval` pre-downloads the following on the login node so +`oellm-eval schedule` pre-downloads the following on the login node so compute nodes do not need internet access: | Asset | HF repo | Size | @@ -99,13 +99,13 @@ Three task groups are available: ```bash # Both splits -oellm schedule-eval \ +oellm-eval schedule \ --models lmsdss/RegionReasoner-7B \ --task-groups regiondial-bench \ --venv-path ~/elliot-venv # Single split -oellm schedule-eval \ +oellm-eval schedule \ --models lmsdss/RegionReasoner-7B \ --task-groups regiondial-refcocog \ --venv-path ~/elliot-venv @@ -114,7 +114,7 @@ oellm schedule-eval \ ### Collecting results ```bash -oellm collect-results \ +oellm-eval collect \ --eval-output-dir /path/to/evals \ --output-csv results.csv ``` @@ -132,7 +132,7 @@ which is detected automatically from the model name. To evaluate a different model, just pass it to `--models`: ```bash -oellm schedule-eval \ +oellm-eval schedule \ --models Qwen/Qwen2.5-VL-7B-Instruct \ --task-groups regiondial-bench \ --venv-path ~/elliot-venv @@ -150,7 +150,7 @@ The model type is resolved as follows: To evaluate multiple models in one go: ```bash -oellm schedule-eval \ +oellm-eval schedule \ --models "lmsdss/RegionReasoner-7B,Qwen/Qwen2.5-VL-7B-Instruct" \ --task-groups regiondial-bench \ --venv-path ~/elliot-venv diff --git a/oellm/main.py b/oellm/main.py index a6a7ef62..29523516 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -21,7 +21,7 @@ rich_utils.STYLE_OPTION_DEFAULT = "dim" app = typer.Typer( - name="oellm", + name="oellm-eval", help="ELLIOT: Multi-cluster evaluation tool for language models", no_args_is_help=True, pretty_exceptions_show_locals=False, @@ -84,7 +84,7 @@ def schedule_evals( directory shipped with the package, which overrides broken upstream tasks (e.g. mgsm_native_cot_fr/de/es). Override to point at additional task YAMLs. local: If True, run evaluations directly on the local machine using bash instead of - submitting to SLURM. Requires --venv_path. Skips cluster environment detection and + submitting to SLURM. Requires --venv-path. Skips cluster environment detection and runs all evaluations sequentially in a single process. slurm_template_var: JSON object of template variable overrides. Use exact env var names (PARTITION, ACCOUNT, GPUS_PER_NODE, SLURM_MEM). "TIME" overrides the time limit. @@ -297,7 +297,7 @@ def eval_command( ) -> None: """Run evaluations from a YAML config file. - All CLI flags override values in --config. Delegates to schedule-eval. + All CLI flags override values in --config. Delegates to schedule. Args: config: Path to a YAML config file. @@ -341,9 +341,9 @@ def eval_command( # Register CLI commands -app.command("schedule-eval")(schedule_evals) +app.command("schedule")(schedule_evals) app.command("eval")(eval_command) -app.command("collect-results")(collect_results) +app.command("collect")(collect_results) app.command("list-tasks")(list_tasks) app.command("compare")(compare) diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 232a526b..e2fc0a44 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -242,7 +242,7 @@ do --output_path "$RESULTS_SUBDIR" \ ${{LIMIT:+--limit $LIMIT}} else - echo "[error] evalchemy suite requires --venv_path (not supported in container mode)." + echo "[error] evalchemy suite requires --venv-path (not supported in container mode)." exit 1 fi ;; diff --git a/oellm/results.py b/oellm/results.py index 781d9a5d..37ce33fa 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -538,7 +538,7 @@ def collect_results( missing_df.to_csv(missing_csv, index=False) logging.info(f"Missing jobs saved to: {missing_csv}") logging.info( - f"You can run these with: oellm schedule-eval --eval-csv-path {missing_csv}" + f"You can run these with: oellm-eval schedule --eval-csv-path {missing_csv}" ) if verbose and len(missing_jobs) > 0: diff --git a/oellm/scheduler.py b/oellm/scheduler.py index c298c4df..c8c39064 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -153,7 +153,7 @@ def schedule_evals( Passed as --include_path to lm_eval. Defaults to the bundled custom_lm_eval_tasks directory shipped with the package. local: If True, run evaluations directly on the local machine using bash instead of - submitting to SLURM. Requires --venv_path. + submitting to SLURM. Requires --venv-path. slurm_template_var: JSON object of template variable overrides. Use exact env var names (PARTITION, ACCOUNT, GPUS_PER_NODE, SLURM_MEM). "TIME" overrides the time limit. Example: '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00","GPUS_PER_NODE":2,"SLURM_MEM":"96G"}' @@ -163,7 +163,7 @@ def schedule_evals( if local: if not venv_path: raise ValueError( - "--local requires --venv_path. Provide a path to a Python virtual " + "--local requires --venv-path. Provide a path to a Python virtual " "environment with lm_eval/lighteval installed." ) local_output = str(Path.cwd() / "oellm-output") diff --git a/oellm/utils.py b/oellm/utils.py index eb00e77e..f9975778 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -680,7 +680,7 @@ def check_judge_llm_pre_flight( if allow_missing: logging.warning( "Scheduling %d judge-required task(s) without OPENAI_API_KEY: %s. " - "These will emit null performance values in collect-results.", + "These will emit null performance values in collect.", len(needed), ", ".join(needed), ) diff --git a/pyproject.toml b/pyproject.toml index 31967faf..b2f1030f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "oellm" +name = "oellm-eval" version = "0.1.0" description = "OpenEuroLLM CLI" readme = "README.md" @@ -94,7 +94,7 @@ dclm = [ ] [project.scripts] -oellm = "oellm.main:main" +oellm-eval = "oellm.main:main" [build-system] requires = ["uv_build>=0.7.19,<0.8.0"] diff --git a/tests/integration/test_slurm.py b/tests/integration/test_slurm.py index b6885a03..a9380949 100644 --- a/tests/integration/test_slurm.py +++ b/tests/integration/test_slurm.py @@ -75,17 +75,17 @@ def run_schedule_eval( "schedule", "--models", "HuggingFaceTB/SmolLM2-135M-Instruct", - "--task_groups", + "--task-groups", task_groups, "--limit", str(limit), ] if venv_path: - cmd.extend(["--venv_path", venv_path]) + cmd.extend(["--venv-path", venv_path]) if dry_run: - cmd.extend(["--dry_run", "true"]) + cmd.extend(["--dry-run"]) if skip_checks: - cmd.extend(["--skip_checks", "true"]) + cmd.extend(["--skip-checks"]) return subprocess.run(cmd, capture_output=True, text=True) @@ -102,17 +102,17 @@ def run_schedule_eval_with_csv( "run", "oellm-eval", "schedule", - "--eval_csv_path", + "--eval-csv-path", csv_path, "--limit", str(limit), ] if venv_path: - cmd.extend(["--venv_path", venv_path]) + cmd.extend(["--venv-path", venv_path]) if dry_run: - cmd.extend(["--dry_run", "true"]) + cmd.extend(["--dry-run"]) if skip_checks: - cmd.extend(["--skip_checks", "true"]) + cmd.extend(["--skip-checks"]) return subprocess.run(cmd, capture_output=True, text=True) From f856ef633fa0b0dceff9fb1eaad38112ec645d0a Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Wed, 10 Jun 2026 15:10:38 +0200 Subject: [PATCH 25/44] fixes --- .gitignore | 6 + README.md | 50 ++- docs/CONTAINERS.md | 38 +- docs/LEONARDO.md | 31 +- docs/TASKS.md | 46 ++- docs/VENV.md | 18 +- oellm/config.py | 187 +++++++-- oellm/contrib/CONTRIBUTING.md | 11 +- oellm/contrib/README.md | 18 +- oellm/contrib/audiobench/README.md | 4 +- oellm/contrib/regiondial_bench/README.md | 14 +- oellm/contrib/regiondial_bench/suite.py | 87 +++-- oellm/envcheck.py | 464 +++++++++++++++++++++++ oellm/main.py | 94 ++++- oellm/resources/template.sbatch | 49 ++- oellm/results.py | 38 +- oellm/scheduler.py | 38 +- oellm/task_groups.py | 12 +- oellm/utils.py | 24 +- tests/test_collect_results.py | 117 ++++++ tests/test_envcheck.py | 277 ++++++++++++++ tests/test_eval_config.py | 103 +++++ tests/test_judge_preflight.py | 73 ++++ tests/test_metric_snapshots.py | 205 ++++++++++ tests/test_regiondial_bench.py | 161 ++++++++ tests/test_schedule_evals.py | 116 +++++- 26 files changed, 2117 insertions(+), 164 deletions(-) create mode 100644 oellm/envcheck.py create mode 100644 tests/test_envcheck.py create mode 100644 tests/test_judge_preflight.py create mode 100644 tests/test_metric_snapshots.py diff --git a/.gitignore b/.gitignore index 16c2ad20..f0b25858 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,9 @@ **/task_map_cache.json *.sif results/ +# SSH keys/certs must never be committed +key_filename* +*.pem +id_rsa* +id_ecdsa* +id_ed25519* diff --git a/README.md b/README.md index 1ad10e07..9f44260b 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,26 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Schedule evaluations** on multiple models and tasks: `oellm-eval schedule` - **Collect results** and check for missing evaluations: `oellm-eval collect` +- **Diagnose your environment** (cluster vars, HF cache, venv engines): `oellm-eval doctor` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading -- **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Snellius) +- **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Jupiter, Snellius) - **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) - **Video evaluation** via lmms-eval (VideoMMMU, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench) - **Audio evaluation** via lmms-eval (LibriSpeech, FLEURS, GigaSpeech, TED-LIUM, WenetSpeech, CoVoST2, VocalSound, MuChoMusic) - **Plugin system** for contributing custom benchmarks without touching core code - **Automatic building and deployment of containers** +## Commands at a Glance + +| Command | What it does | +|---|---| +| `oellm-eval schedule` | Expand models × tasks, pre-download models/datasets on the login node, generate and submit a SLURM array job (or run locally with `--local`) | +| `oellm-eval eval --config eval.yaml` | Same as `schedule`, driven by a YAML config file; CLI flags override the file | +| `oellm-eval collect ` | Aggregate result JSONs into `eval_results.csv` + `.json` + `.md`; `--check` writes a re-schedulable CSV of missing jobs | +| `oellm-eval list-tasks` | Show every task group, its engine, task count, and n-shot settings | +| `oellm-eval compare ` | Diff two collected `results.json` files task by task | +| `oellm-eval doctor` | Diagnose the environment: cluster detection, env vars, HF cache, venv engines | + ## Quick Start **Prerequisites:** @@ -37,7 +49,7 @@ oellm-eval schedule \ ``` This will automatically: -- Detect your current HPC cluster (Leonardo, LUMI, JURECA, or Snellius) +- Detect your current HPC cluster (Leonardo, LUMI, JURECA, Jupiter, or Snellius) - Download and cache the specified models - Pre-download datasets for known tasks (see warning below) - Generate and submit a SLURM job array with appropriate cluster-specific resources and using containers built for this cluster @@ -88,7 +100,7 @@ Super groups: `oellm-multilingual` (all multilingual benchmarks combined) | `video-activitynet-qa` | ActivityNet-QA (requires GPT API) | lmms-eval | | `video-longvideobench` | LongVideoBench (cross-segment reasoning) | lmms-eval | -The lmms-eval adapter class (`llava_hf`, `llava_onevision`, `qwen2_5_vl`, etc.) is auto-detected from the model name. Install with `pip install oellm[video]` (or use a venv with lmms-eval). +The lmms-eval adapter class (`llava_hf`, `llava_onevision`, `qwen2_5_vl`, etc.) is auto-detected from the model name. Video (like image and audio) tasks run through **lmms-eval, which is not included in the cluster containers or any pip extra** — set up the general venv as described in [docs/VENV.md](docs/VENV.md) and pass `--venv-path`. ### Audio @@ -110,7 +122,7 @@ The lmms-eval adapter class (`llava_hf`, `llava_onevision`, `qwen2_5_vl`, etc.) | `audio-alpaca-audio`, `audio-openhermes`, `audio-wavcaps` | Instruction / captioning (GPT judge) | lmms-eval | | `audio-clotho-aqa`, `audio-cn-college-listen-mcq`, `audio-dream-tts-mcq`, `audio-voicebench`, `audio-step2-paralinguistic` | QA / MCQ / paralinguistic probes | lmms-eval | -Install with `pip install oellm[audio]`. Judge-model groups (AIR-Bench chat, Alpaca-Audio, OpenHermes, WavCaps) need `OPENAI_API_KEY` on the compute node. The HPC Singularity image must include `ffmpeg` for non-WAV decode. +Audio tasks also run through lmms-eval — use the general venv from [docs/VENV.md](docs/VENV.md) (the `[audio]` extra adds the audio decoding helpers, but lmms-eval itself must be installed per that guide). Judge-model groups (AIR-Bench chat, Alpaca-Audio, OpenHermes, WavCaps) need `OPENAI_API_KEY` on the compute node — scheduling refuses without it unless you pass `--allow-missing-judge`. The HPC Singularity image must include `ffmpeg` for non-WAV decode. ### Custom Benchmarks (contrib) @@ -163,7 +175,7 @@ Results are written to `./oellm-output//results/`. **Air-gapped cluster nodes (no internet):** batch jobs set `HF_HUB_OFFLINE=1` and get `HF_HOME` from your cluster env. With `--local`, the CLI defaults `HF_HOME` to `~/.cache/huggingface` if unset and would otherwise allow Hub access—so on a compute node without network, export your real cache and offline flag before running, for example: ```bash -export HF_HOME=/leonardo_work/OELLM_prod2026/users/shaldar0/oellm-evals/hf_data +export HF_HOME=/path/to/your/shared/hf_cache # e.g. $WORK/hf_cache on Leonardo export HF_HUB_OFFLINE=1 oellm-eval schedule ... --venv-path .venv --local ``` @@ -217,6 +229,7 @@ MODEL_ARGS='batch_size=8' oellm-eval schedule \ If you use custom tasks via `--tasks` that are not in the task groups registry, the CLI will attempt to look them up but **cannot guarantee the datasets will be cached**. This may cause failures on compute nodes that don't have network access. **Recommendation:** Use `--task-groups` when possible, or ensure your custom task datasets are already cached in `$HF_HOME` before scheduling. + ## Collecting Results After evaluations complete, collect results into a CSV. `collect` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs: @@ -235,13 +248,15 @@ output/ ``` ```bash -# Basic collection +# Basic collection — writes eval_results.csv, eval_results.json, eval_results.md oellm-eval collect /path/to/eval-output-dir # Check for missing evaluations and create a CSV for re-running them oellm-eval collect /path/to/eval-output-dir --check --output-csv results.csv ``` +Three output files are written next to your `--output-csv` path: the CSV (raw metric per row), a versioned JSON envelope, and a Markdown table with metrics normalized to a 0–100 scale. + All `jobs.csv` files found under `results_dir` are merged into one; if the same `(model_path, task_path, n_shot)` row appears in multiple files the later-sorted entry wins (override duplicates). The merged jobs list is then compared against all `.json` result files found recursively. The `--check` flag outputs a `results_missing.csv` that can be used to re-schedule failed jobs: @@ -252,12 +267,14 @@ oellm-eval schedule --eval-csv-path results_missing.csv ## CSV-Based Scheduling -For full control, provide a CSV file with columns: `model_path`, `task_path`, `n_shot`, and optionally `eval_suite`: +For full control, provide a CSV file with columns: `model_path`, `task_path`, `n_shot`, and optionally `eval_suite` (one of `lm_eval` — the default, `lighteval`, `lmms_eval`, `evalchemy`, or a contrib suite name): ```bash oellm-eval schedule --eval-csv-path custom_evals.csv ``` +> **Note:** field values must not contain commas, quotes, or newlines — the SLURM-side reader splits rows on commas, and scheduling rejects such rows with an error. Model args like `model,revision=...` are not supported. + ## Installation ```bash @@ -280,8 +297,23 @@ export UV_PYTHON_INSTALL_DIR="/p/project1//$USER/.local/share/uv/python export UV_TOOL_DIR="/p/project1//$USER/.cache/uv-tool-cache" ``` -## Supported Clusters: -We support: Leonardo, Lumi, Jureca, Jupiter, and Snellius +## Supported Clusters + +Leonardo, LUMI, JURECA, Jupiter, and Snellius — detected automatically from the login node's hostname (see [`oellm/resources/clusters.yaml`](oellm/resources/clusters.yaml)). Any value there can be overridden by exporting the environment variable before scheduling. + +## Environment Diagnostics + +`schedule` verifies before submission that the chosen runtime (venv or container) can actually run the scheduled suites — missing engines, missing suite env vars (e.g. `AUDIOBENCH_DIR`), and version-pinned groups on the wrong engine (`dclm-core-22` needs `lm-eval==0.4.9.2`) are rejected with an actionable message instead of failing hours later on a compute node. Bypass with `--skip-checks`. + +Run the same checks standalone at any time: + +```bash +# Full report: cluster detection, env vars, HF cache, SLURM binaries, venv engines +oellm-eval doctor --venv-path /path/to/.venv + +# Treat the engines these groups need as required (exit 1 if missing) +oellm-eval doctor --venv-path /path/to/.venv --task-groups "image-vqa,open-sci-0.01" +``` ## CLI Options diff --git a/docs/CONTAINERS.md b/docs/CONTAINERS.md index e2afbd94..bd33c45e 100644 --- a/docs/CONTAINERS.md +++ b/docs/CONTAINERS.md @@ -7,29 +7,43 @@ Apptainer containers are built automatically via GitHub Actions and stored on Hu ## How It Works 1. Definition files live in `containers/.def` -2. On push to `main` (when `.def` files change), GitHub Actions provisions [Lambda Labs](https://lambdalabs.com/) GPU instances via [SkyPilot](https://skypilot.readthedocs.io/) and builds all containers in parallel +2. Builds are triggered **manually** ("Run workflow" on the [build-and-push-apptainer](../.github/workflows/build-and-push-apptainer.yml) action — manual since PR #46). The workflow provisions [Lambda Labs](https://lambdalabs.com/) GPU instances via [SkyPilot](https://skypilot.readthedocs.io/) and builds all containers in parallel 3. Built `.sif` images are uploaded to HuggingFace Hub 4. Clusters pull the image specified in `oellm/resources/clusters.yaml` via `EVAL_CONTAINER_IMAGE` Images are compressed with zstd (level 3) via mksquashfs for a good balance of size and build speed. -## Image Evaluation (lmms-eval) +## What the Images Contain (and What They Don't) -Image benchmarks (`suite: lmms_eval`) require `lmms-eval` to be available in the execution environment. There are two ways to provide it: +The shipped images provide exactly **two engines**: `lm-eval` (system Python) +and `lighteval` (installed as an isolated uv tool to avoid the `datasets` +version conflict). They do **not** contain `lmms-eval`, the `oellm` package +(needed by contrib suites), or evalchemy — scheduling those suites in +container mode is rejected by the environment pre-flight; pass `--venv-path` +with a suitable venv instead (see [VENV.md](VENV.md)). -**Option 1 — Custom venv (recommended for development):** -Install `lmms-eval` via the `[image]` (or `[audio]`/`[video]`) pyproject extra and pass `--venv-path` to the CLI: -```bash -uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' -``` -See [VENV.md](VENV.md) for full setup instructions. +## Image / Video / Audio Evaluation (lmms-eval) + +lmms-eval benchmarks require a custom venv: + +**Option 1 — Custom venv (recommended):** +Follow the general-venv setup in [VENV.md](VENV.md) — note that `lmms-eval` +is **not** provided by any pyproject extra; it must be installed editable +from git (its wheel build drops required template files), alongside the +`[text,image,audio]` extras. -**Option 2 — Container with lmms-eval:** -Build a container `.def` file that includes `lmms-eval` alongside `lm-eval`: +**Option 2 — Build a container with lmms-eval:** +Extend a `.def` file — install lmms-eval **editable from git** (a plain +`pip install lmms-eval` produces a broken install — the wheel drops +`_default_template_yaml` files) and include the `oellm` package if contrib +suites should run in the container: ```singularity %post - pip install lm-eval torch transformers accelerate "datasets<4.0.0" "lmms-eval>=0.2.4" + uv pip install --system --break-system-packages \ + lm-eval torch transformers accelerate "datasets<4.0.0" + uv pip install --system --break-system-packages \ + -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git#egg=lmms-eval" ``` Then set `EVAL_CONTAINER_IMAGE` in `clusters.yaml` to point to this image. diff --git a/docs/LEONARDO.md b/docs/LEONARDO.md index 385b7f3e..1242e256 100644 --- a/docs/LEONARDO.md +++ b/docs/LEONARDO.md @@ -29,18 +29,27 @@ step ca bootstrap \ ``` ### Generate an SSH Certificate -Replace `your.name@email.com` with the email you used when registering on UserDB: +Replace `your.name@email.com` with the email you used when registering on UserDB. + +> ⚠️ Generate the key **into `~/.ssh/`**, never into a project directory — a +> private key inside a git repository is one `git add -A` away from being +> committed and leaked. + ```zsh -step ssh certificate your.name@email.com --provisioner cineca-hpc key_filename +step ssh certificate your.name@email.com --provisioner cineca-hpc ~/.ssh/cineca_hpc ``` +This writes the private key `~/.ssh/cineca_hpc` plus `cineca_hpc.pub` and +`cineca_hpc-cert.pub`. The certificate is short-lived (~12 h) — rerun the +command when it expires. + ### Configure SSH Add the following to `~/.ssh/config` (create it if it doesn't exist via `nano ~/.ssh/config`): ``` Host leonardo HostName login07-ext.leonardo.cineca.it User your_cineca_username - IdentityFile /Users/your_mac_username/.ssh/key_filename + IdentityFile ~/.ssh/cineca_hpc ``` Set correct permissions: @@ -145,7 +154,21 @@ export HF_HOME="$SCRATCH/hf_cache" --- -## 6. Running Evaluations +## 6. Verify the Setup + +Before scheduling anything, run the built-in diagnostic — it checks cluster +detection, required environment variables, your HF cache, SLURM binaries, and +(with `--venv-path`) which eval engines your venv actually provides: + +```zsh +oellm-eval doctor +# with a custom venv and the groups you plan to run: +oellm-eval doctor --venv-path /path/to/.venv --task-groups "open-sci-0.01" +``` + +--- + +## 7. Running Evaluations ```zsh # Run evaluations using a task group (recommended) diff --git a/docs/TASKS.md b/docs/TASKS.md index dc0a574e..4d520f80 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -4,13 +4,18 @@ Tasks are defined in `oellm/resources/task-groups.yaml`. Only tasks in this file are tested and guaranteed to work. The CLI parses this via `task_groups.py` and expands groups into `(task, n_shot, suite)` tuples for scheduling. -Three evaluation suites are supported: +Supported evaluation suites: | Suite value | Engine | Use case | |---|---|---| -| `lm_eval` | [lm-eval](https://github.com/EleutherAI/lm-evaluation-harness) | Text benchmarks | +| `lm_eval` (alias `lm-eval-harness`) | [lm-eval](https://github.com/EleutherAI/lm-evaluation-harness) | Text benchmarks | | `lighteval` | [lighteval](https://github.com/huggingface/lighteval) | Translation / multilingual | -| `lmms_eval` | [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) | Image / VQA benchmarks | +| `lmms_eval` | [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) | Image / video / audio benchmarks | +| `evalchemy` | [evalchemy](https://github.com/mlfoundations/evalchemy) (fork) | Free-form reasoning (GPQA, MATH500, LiveCodeBench) — needs its own venv, see [VENV.md](VENV.md) | +| `` | a contrib plugin | Custom benchmarks — see [CONTRIBUTING.md](../oellm/contrib/CONTRIBUTING.md) | + +`suite` is set at the group level and can be overridden per task — the +`reasoning` group uses this to mix lm-eval and evalchemy tasks in one group. ## YAML Structure @@ -83,19 +88,34 @@ from the model name. No manual override is needed. | Field | Required | Level | Description | |-------|----------|-------|-------------| | `description` | Yes | group | Short description of the task group | -| `suite` | Yes | group | Evaluation suite: `lm_eval`, `lighteval`, or `lmms_eval` | +| `suite` | Yes (group) | group or task | Evaluation suite; a task-level value overrides the group | | `n_shots` | Yes | group or task | List of shot counts; must be set at group or task level | -| `dataset` | Yes | group or task | HuggingFace dataset repo ID (required for pre-download and testing) | +| `dataset` | Recommended | group or task | HuggingFace dataset repo ID — without it, nothing is pre-downloaded for the task | | `task` | Yes | task | Task name as recognized by the evaluation suite | | `subset` | No | task | HuggingFace dataset config/subset name | - -## Important: Dataset Requirement - -**You must provide the `dataset` field** (at group or task level) for: -1. **Automatic pre-download** - Compute nodes often lack network access; datasets are cached beforehand -2. **CI testing** - The test suite validates that all datasets in `task-groups.yaml` are accessible - -Tasks without a `dataset` field will not have their data pre-downloaded and are not covered by CI validation. +| `revisions` | No | task | HF dataset revisions/branches to pre-fetch (default `["main"]`; e.g. MVBench keeps videos on a `video` branch) | +| `hf_models` | No | task | Auxiliary HF *model* repos to pre-download (e.g. a task router or SAM checkpoint) | +| `hf_dataset_files` | No | task | Specific files to fetch from a dataset repo: `{repo_id, patterns, revision?}` — use for large repos where only a subset is needed | + +## Important: Dataset Pre-Download Behavior + +**Provide the `dataset` field** (at group or task level) wherever possible: +compute nodes run with `HF_HUB_OFFLINE=1`, so anything not cached on the +login node before submission fails at eval time. Tasks without a `dataset` +field are simply skipped by the pre-download step. + +Two details worth knowing: + +1. **The group-name prefix selects the download strategy.** Groups whose name + starts with `audio-`, `video-`, or `image-` are fetched with + `snapshot_download` (raw repo files; the compute node builds the dataset at + runtime — this avoids out-of-memory kills on the login node for large + media datasets). All other groups go through `load_dataset()`. If you add + a media group, keep the prefix. +2. **Dataset accessibility is verified by `tests/test_datasets.py`**, which + needs network access and is therefore excluded from the offline CI run — + execute it locally when adding datasets: + `uv run pytest tests/test_datasets.py -k my-group`. ## Custom Benchmarks (contrib plugins) diff --git a/docs/VENV.md b/docs/VENV.md index 8748131d..7b2e6703 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -24,7 +24,9 @@ documented in `oellm/contrib//README.md`: | `audio-audiobench*` | `audiobench` | [`oellm/contrib/audiobench/README.md`](../oellm/contrib/audiobench/README.md) | | `regiondial-*` | `regiondial_bench` | [`oellm/contrib/regiondial_bench/README.md`](../oellm/contrib/regiondial_bench/README.md) | -Use `oellm list-tasks` to see which suite a given task group routes to. +Use `oellm-eval list-tasks` to see which suite a given task group routes to, +and `oellm-eval doctor --venv-path --task-groups ` to verify a +venv against the groups you plan to run. ## Setup (general venv) @@ -32,8 +34,10 @@ Use `oellm list-tasks` to see which suite a given task group routes to. # 1. Create venv uv venv --python 3.12 /path/to/.venv -# 2. Install lmms-eval editable from main +# 2. Install lmms-eval editable from git # (Editable is required — wheel build drops `_default_template_yaml` files.) +# Pin a known-good commit (`...lmms-eval.git@#egg=...`) so two venvs +# created on different days run the same engine — unpinned `main` drifts. uv pip install --python /path/to/.venv/bin/python \ -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git#egg=lmms-eval" @@ -49,9 +53,9 @@ UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ Verify with: ```bash -/path/to/.venv/bin/python -c \ - 'from lmms_eval.tasks import TaskManager; TaskManager("INFO"); print("OK")' +oellm-eval doctor --venv-path /path/to/.venv --task-groups "open-sci-0.01,image-vqa" ``` +(or manually: `/path/to/.venv/bin/python -c 'from lmms_eval.tasks import TaskManager; TaskManager("INFO"); print("OK")'`) ## Usage @@ -98,9 +102,13 @@ oellm-eval schedule \ --models Qwen/Qwen3-0.6B-Base \ --task-groups dclm-core-22 \ --venv-path dclm-core-venv \ - --skip-checks + --skip-checks # skips dataset pre-download AND the environment pre-flight — make sure datasets are already cached ``` +> Without `--skip-checks`, the scheduler verifies the venv actually contains +> `lm-eval==0.4.9.2` — running this group on any other lm-eval version +> silently changes the scores. + ## Evalchemy (reasoning) The `reasoning` task group includes 6 benchmarks: GSM8k, IFEval, and MBPP run via lm-eval-harness, while GPQADiamond, MATH500, and LiveCodeBench run via evalchemy. diff --git a/oellm/config.py b/oellm/config.py index 70e6bf35..959badfc 100644 --- a/oellm/config.py +++ b/oellm/config.py @@ -26,10 +26,14 @@ class SlurmOverrides: gpus_per_node: int | None = None time_limit: str | None = None max_array_len: int = 128 + # Template vars the dataclass doesn't model (SLURM_MEM, CPUS_PER_TASK, …). + # The scheduler applies any key to the environment, so these must survive + # the EvalConfig round-trip verbatim instead of being dropped. + extra_template_vars: dict[str, str] = field(default_factory=dict) def to_template_var_dict(self) -> dict[str, str]: """Return a dict suitable for ``slurm_template_var`` JSON consumption.""" - d: dict[str, str] = {} + d: dict[str, str] = dict(self.extra_template_vars) if self.partition is not None: d["PARTITION"] = self.partition if self.account is not None: @@ -84,6 +88,12 @@ class EvalConfig: # ---- SLURM overrides ---- slurm: SlurmOverrides = field(default_factory=SlurmOverrides) + # Set by from_cli_kwargs: names of fields explicitly provided on the CLI + # ("slurm.max_array_len" for the nested one). Plain class attribute, not a + # dataclass field, so it stays out of __init__/fields(). merge() uses it + # to distinguish "--no-dry-run" from "flag not given". + _cli_provided = None + # ------------------------------------------------------------------ # Construction helpers # ------------------------------------------------------------------ @@ -125,22 +135,24 @@ def from_cli_kwargs( task_groups: str | None = None, n_shot: int | list[int] | None = None, eval_csv_path: str | None = None, - max_array_len: int = 128, + max_array_len: int | None = None, limit: int | None = None, - verbose: bool = False, - download_only: bool = False, - dry_run: bool = False, - skip_checks: bool = False, - trust_remote_code: bool = True, + verbose: bool | None = None, + download_only: bool | None = None, + dry_run: bool | None = None, + skip_checks: bool | None = None, + trust_remote_code: bool | None = None, venv_path: str | None = None, lm_eval_include_path: str | None = None, - local: bool = False, + local: bool | None = None, slurm_template_var: str | None = None, ) -> EvalConfig: """Build an ``EvalConfig`` from the loose CLI parameters. - This is the bridge that keeps the existing CLI signature 100 % - backward-compatible. + Every parameter defaults to ``None`` meaning "not provided on the CLI". + Provided parameters are recorded so :meth:`merge` can let an explicit + CLI value override YAML even when it equals the class default + (e.g. ``--no-dry-run`` against ``dry_run: true``). """ import json @@ -165,7 +177,30 @@ def from_cli_kwargs( elif isinstance(n_shot, list): n_shot_list = n_shot - slurm = SlurmOverrides(max_array_len=max_array_len) + provided: set[str] = set() + for field_name, value in ( + ("models", models_list), + ("tasks", tasks_list), + ("task_groups", groups_list), + ("n_shot", n_shot_list), + ("eval_csv_path", eval_csv_path), + ("limit", limit), + ("verbose", verbose), + ("download_only", download_only), + ("dry_run", dry_run), + ("skip_checks", skip_checks), + ("trust_remote_code", trust_remote_code), + ("venv_path", venv_path), + ("lm_eval_include_path", lm_eval_include_path), + ("local", local), + ): + if value is not None: + provided.add(field_name) + + slurm = SlurmOverrides() + if max_array_len is not None: + slurm.max_array_len = max_array_len + provided.add("slurm.max_array_len") if slurm_template_var: try: opts = json.loads(slurm_template_var) @@ -178,35 +213,84 @@ def from_cli_kwargs( "slurm_template_var must be a JSON object, e.g. " '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00"}' ) - slurm.partition = opts.get("PARTITION", opts.get("partition")) - slurm.account = opts.get("ACCOUNT", opts.get("account")) - gpus = opts.get("GPUS_PER_NODE", opts.get("gpus_per_node")) - if gpus is not None: - slurm.gpus_per_node = int(gpus) - slurm.time_limit = opts.get("TIME", opts.get("time_limit")) - - return cls( + for key, value in opts.items(): + upper = str(key).upper() + if upper == "PARTITION": + slurm.partition = value + elif upper == "ACCOUNT": + slurm.account = value + elif upper == "GPUS_PER_NODE": + slurm.gpus_per_node = int(value) + elif upper == "TIME" or key == "time_limit": + slurm.time_limit = str(value) + else: + # Unmodeled keys (SLURM_MEM, …) pass through verbatim — the + # scheduler exports any key into the job environment. + slurm.extra_template_vars[str(key)] = str(value) + + cfg = cls( models=models_list, tasks=tasks_list, task_groups=groups_list, n_shot=n_shot_list, eval_csv_path=eval_csv_path, limit=limit, - verbose=verbose, - download_only=download_only, - dry_run=dry_run, - skip_checks=skip_checks, - trust_remote_code=trust_remote_code, + verbose=bool(verbose) if verbose is not None else False, + download_only=bool(download_only) if download_only is not None else False, + dry_run=bool(dry_run) if dry_run is not None else False, + skip_checks=bool(skip_checks) if skip_checks is not None else False, + trust_remote_code=bool(trust_remote_code) + if trust_remote_code is not None + else True, venv_path=venv_path, lm_eval_include_path=lm_eval_include_path, - local=local, + local=bool(local) if local is not None else False, slurm=slurm, ) + cfg._cli_provided = provided + return cfg + + _KNOWN_KEYS = frozenset( + { + "models", + "tasks", + "task_groups", + "n_shot", + "eval_csv_path", + "limit", + "verbose", + "download_only", + "dry_run", + "skip_checks", + "trust_remote_code", + "venv_path", + "lm_eval_include_path", + "local", + "slurm", + } + ) + _KNOWN_SLURM_KEYS = frozenset( + {"partition", "account", "gpus_per_node", "time_limit", "max_array_len"} + ) @classmethod def _from_dict(cls, raw: dict[str, Any]) -> EvalConfig: """Construct from a raw dict (YAML or programmatic).""" + unknown = set(raw) - cls._KNOWN_KEYS + if unknown: + logging.warning( + f"Ignoring unknown config key(s): {', '.join(sorted(unknown))}. " + f"Known keys: {', '.join(sorted(cls._KNOWN_KEYS))}." + ) + slurm_raw = raw.get("slurm", {}) or {} + unknown_slurm = set(slurm_raw) - cls._KNOWN_SLURM_KEYS + if unknown_slurm: + logging.warning( + f"Ignoring unknown slurm config key(s): " + f"{', '.join(sorted(unknown_slurm))}. " + f"Known keys: {', '.join(sorted(cls._KNOWN_SLURM_KEYS))}." + ) slurm = SlurmOverrides( partition=slurm_raw.get("partition"), account=slurm_raw.get("account"), @@ -242,8 +326,15 @@ def _from_dict(cls, raw: dict[str, Any]) -> EvalConfig: def merge(self, cli: EvalConfig) -> EvalConfig: """Return a new config where *cli* values override *self* (the YAML base). - A CLI field is considered "set" when it differs from the class default. + When *cli* was built by :meth:`from_cli_kwargs`, "set" means the flag + was actually provided (tracked explicitly), so a CLI value equal to + the class default still overrides YAML (e.g. ``--no-dry-run`` beats + ``dry_run: true``). For configs constructed directly (without the + tracking attribute), falls back to the legacy heuristic of comparing + against the class default. """ + provided: set[str] | None = getattr(cli, "_cli_provided", None) + merged_kwargs: dict[str, Any] = {} for f in fields(self): yaml_val = getattr(self, f.name) @@ -251,9 +342,11 @@ def merge(self, cli: EvalConfig) -> EvalConfig: default_val = _field_default(f) if f.name == "slurm": - merged_kwargs["slurm"] = _merge_slurm(yaml_val, cli_val) + merged_kwargs["slurm"] = _merge_slurm(yaml_val, cli_val, provided) + elif provided is not None: + merged_kwargs[f.name] = cli_val if f.name in provided else yaml_val elif cli_val != default_val: - # CLI explicitly set — use it + # Legacy heuristic: CLI considered set when ≠ class default merged_kwargs[f.name] = cli_val else: merged_kwargs[f.name] = yaml_val @@ -379,23 +472,41 @@ def _field_default(f: Any) -> Any: return None -def _merge_slurm(yaml_slurm: SlurmOverrides, cli_slurm: SlurmOverrides) -> SlurmOverrides: - """Merge two SlurmOverrides — CLI wins when non-default.""" - default = SlurmOverrides() +def _merge_slurm( + yaml_slurm: SlurmOverrides, + cli_slurm: SlurmOverrides, + provided: set[str] | None = None, +) -> SlurmOverrides: + """Merge two SlurmOverrides — CLI wins when set (None means unset).""" + if provided is not None: + max_array_len = ( + cli_slurm.max_array_len + if "slurm.max_array_len" in provided + else yaml_slurm.max_array_len + ) + else: + # Legacy heuristic for configs without provenance tracking + max_array_len = ( + cli_slurm.max_array_len + if cli_slurm.max_array_len != SlurmOverrides().max_array_len + else yaml_slurm.max_array_len + ) return SlurmOverrides( partition=cli_slurm.partition - if cli_slurm.partition != default.partition + if cli_slurm.partition is not None else yaml_slurm.partition, account=cli_slurm.account - if cli_slurm.account != default.account + if cli_slurm.account is not None else yaml_slurm.account, gpus_per_node=cli_slurm.gpus_per_node - if cli_slurm.gpus_per_node != default.gpus_per_node + if cli_slurm.gpus_per_node is not None else yaml_slurm.gpus_per_node, time_limit=cli_slurm.time_limit - if cli_slurm.time_limit != default.time_limit + if cli_slurm.time_limit is not None else yaml_slurm.time_limit, - max_array_len=cli_slurm.max_array_len - if cli_slurm.max_array_len != default.max_array_len - else yaml_slurm.max_array_len, + max_array_len=max_array_len, + extra_template_vars={ + **yaml_slurm.extra_template_vars, + **cli_slurm.extra_template_vars, + }, ) diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index 9010aec6..e968b123 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -207,7 +207,16 @@ def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: return None ``` -`CLUSTER_ENV_VARS` are validated by `dispatch.py` before `run()` is called. +`CLUSTER_ENV_VARS` are validated twice: by the scheduler's environment +pre-flight on the login node (before SLURM submission) and by `dispatch.py` +on the compute node before `run()` is called. + +> **Note on `parse_results`:** the protocol requires it (and the registry +> tests enforce it), but result collection currently parses the +> lmms-eval-shaped JSON that `run()` writes *generically* — `parse_results` +> is not invoked by `collect`. Treat the JSON shape written by `run()` as the +> real contract; keep `parse_results` correct so the suite is ready for +> format-specific collection. --- diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md index 917d8f23..a25f5628 100644 --- a/oellm/contrib/README.md +++ b/oellm/contrib/README.md @@ -9,6 +9,7 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). | Benchmark | Task Group | Description | Paper | Code | |---|---|---|---|---| | RegionDial-Bench | `regiondial-bench` | Multi-round region grounding and segmentation on RefCOCOg and RefCOCO+. Evaluates robustness to error accumulation across dialogue turns. | [arXiv:2602.03733](https://arxiv.org/abs/2602.03733) | [lmsdss/RegionReasoner](https://github.com/lmsdss/RegionReasoner) | +| AudioBench | `audio-audiobench` (+ `-asr` / `-st` / `-reasoning`) | 27 judge-free audio tasks — ASR (WER), speech translation (BLEU), spoken reasoning, AudioCaps captioning — scored with AudioBench's own normalisers for paper-comparable numbers. | [arXiv:2406.16020](https://arxiv.org/abs/2406.16020) | [AudioLLMs/AudioBench](https://github.com/AudioLLMs/AudioBench) | ### RegionDial-Bench @@ -18,7 +19,20 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). oellm-eval schedule \ --models lmsdss/RegionReasoner-7B \ --task-groups regiondial-bench \ - --venv-path ~/elliot-venv + --venv-path ~/regiondial-venv ``` -Requires cluster-specific setup (`REGION_REASONER_DIR`, etc.). See the full [RegionDial-Bench README](regiondial_bench/README.md) for prerequisites and configuration. +Requires cluster-specific setup (`REGION_REASONER_DIR`, a dedicated venv, ~30 GB of HF cache). See the full [RegionDial-Bench README](regiondial_bench/README.md) for prerequisites and configuration. + +### AudioBench + +**Metrics:** `wer` (ASR), `bleu` (speech translation), `accuracy` / `string_match` (reasoning), `meteor` (captioning) + +```bash +oellm-eval schedule \ + --models Qwen/Qwen2-Audio-7B-Instruct \ + --task-groups audio-audiobench \ + --venv-path ~/audiobench-venv +``` + +Requires cluster-specific setup (`AUDIOBENCH_DIR` pointing at an AudioBench clone, a dedicated venv). Only the model families AudioBench itself supports can be evaluated (Qwen2-Audio, SALMONN, Whisper, …). See the full [AudioBench README](audiobench/README.md) for prerequisites and the supported-model table. diff --git a/oellm/contrib/audiobench/README.md b/oellm/contrib/audiobench/README.md index d51590b8..6276acdd 100644 --- a/oellm/contrib/audiobench/README.md +++ b/oellm/contrib/audiobench/README.md @@ -141,9 +141,7 @@ unset, the full test split is evaluated. ### Collecting results ```bash -oellm-eval collect \ - --eval-output-dir /path/to/evals \ - --output-csv audiobench_results.csv +oellm-eval collect /path/to/evals --output-csv audiobench_results.csv ``` The primary metric per task is what's registered in `task_metrics` diff --git a/oellm/contrib/regiondial_bench/README.md b/oellm/contrib/regiondial_bench/README.md index fea13252..d761da27 100644 --- a/oellm/contrib/regiondial_bench/README.md +++ b/oellm/contrib/regiondial_bench/README.md @@ -114,14 +114,13 @@ oellm-eval schedule \ ### Collecting results ```bash -oellm-eval collect \ - --eval-output-dir /path/to/evals \ - --output-csv results.csv +oellm-eval collect /path/to/evals --output-csv results.csv ``` -The primary metric in the CSV is **gIoU**. Per-round metrics (e.g. -`gIoU_R1`, `bbox_AP_R3`) are included when the inference script outputs -a `round` field per sample. +The primary metric in the CSV is **gIoU**. Per-round metrics (`gIoU_R1..R7`, +`bbox_AP_R1..R7`) are always computed — rounds are inferred from the order +in which each image's turns appear in the inference output (the script does +not emit an explicit round field). --- @@ -140,9 +139,10 @@ oellm-eval schedule \ The model type is resolved as follows: -| Model name pattern | `--model` flag | +| Model name pattern (checked in order) | `--model` flag | |---|---| | `*regionreasoner*` / `*region_reasoner*` | `vision_reasoner` | +| `*qwen2.5*` | `qwen2.5` | | `*qwen2*` | `qwen2` | | `*qwen*` | `qwen` | | anything else | `vision_reasoner` (default) | diff --git a/oellm/contrib/regiondial_bench/suite.py b/oellm/contrib/regiondial_bench/suite.py index b96cb190..3197c5b9 100644 --- a/oellm/contrib/regiondial_bench/suite.py +++ b/oellm/contrib/regiondial_bench/suite.py @@ -172,12 +172,20 @@ def run( ) with tempfile.TemporaryDirectory(prefix="rr_shards_") as tmp_dir: - shard_paths = _stream_preshard(test_json, tmp_dir, num_gpus) + shard_paths, expected_turns = _stream_preshard(test_json, tmp_dir, num_gpus) procs = [] for idx in range(num_gpus): shard_env = dict(env) shard_env["CUDA_VISIBLE_DEVICES"] = str(idx) + # Each shard gets its own output dir and runs with + # ``--idx 0 --num_parts 1``: the script slices its input with + # ``dataset[idx*part_size : ...]``, so any idx > 0 combined with + # num_parts=1 yields an EMPTY slice (the shard silently evaluates + # nothing). Distinct output dirs replace distinct --idx values as + # the collision guard for the script's output_{idx}.json naming. + shard_out = Path(tmp_dir) / f"shard_{idx}" + shard_out.mkdir() cmd = [ "python", str(inference_script), @@ -188,11 +196,11 @@ def run( "--test_data_path", shard_paths[idx], "--output_path", - tmp_dir, + str(shard_out), "--vis_output_path", - str(Path(tmp_dir) / f"vis_{idx}"), + str(shard_out / "vis"), "--idx", - str(idx), + "0", "--num_parts", "1", "--batch_size", @@ -205,16 +213,20 @@ def run( proc = subprocess.Popen(cmd, env=shard_env, cwd=str(Path(test_json).parent)) procs.append(proc) - for idx, proc in enumerate(procs): - ret = proc.wait() - if ret != 0: - raise RuntimeError( - f"RegionDial-Bench inference shard {idx} exited with code {ret}" - ) + # Wait for ALL shards before raising: raising on the first failure + # would leave sibling GPU processes running while the enclosing + # TemporaryDirectory tears down the files they are writing to. + exit_codes = [proc.wait() for proc in procs] + failed = [(idx, ret) for idx, ret in enumerate(exit_codes) if ret != 0] + if failed: + raise RuntimeError( + "RegionDial-Bench inference shard(s) failed: " + + ", ".join(f"shard {idx} exited with code {ret}" for idx, ret in failed) + ) logger.info("All %d shards completed. Computing metrics.", num_gpus) - metrics = _aggregate_shards(tmp_dir) + metrics = _aggregate_shards(tmp_dir, expected_samples=expected_turns) result_json = { "model_name_or_path": model_path, @@ -227,18 +239,25 @@ def run( logger.info("Results written to %s", output_path) -def _stream_preshard(json_path: str, out_dir: str, num_shards: int) -> list[str]: +def _stream_preshard( + json_path: str, out_dir: str, num_shards: int +) -> tuple[list[str], int]: """Split a large JSON array into *num_shards* files using streaming. Uses ``ijson`` to iterate over the top-level array without loading the - entire file into memory. Items are distributed round-robin. + entire file into memory. Items (whole conversations) are distributed + round-robin — turns within an item always stay together, which the + per-round metric inference in :func:`_aggregate_shards` relies on. - Returns a list of shard file paths. + Returns ``(shard_paths, total_turns)`` where *total_turns* is the number + of per-turn result records the inference run is expected to emit (one per + entry in each item's ``conversational_turns``). """ import ijson shard_files = [] shard_counts = [0] * num_shards + total_turns = 0 for idx in range(num_shards): p = str(Path(out_dir) / f"shard_{idx}.json") shard_files.append(open(p, "w")) # noqa: SIM115 @@ -251,8 +270,12 @@ def _stream_preshard(json_path: str, out_dir: str, num_shards: int) -> list[str] shard_idx = i % num_shards if shard_counts[shard_idx] > 0: shard_files[shard_idx].write(",\n") - json.dump(item, shard_files[shard_idx]) + # ijson yields decimal.Decimal for JSON floats (bbox coordinates); + # json.dump cannot serialize Decimal without a default. + json.dump(item, shard_files[shard_idx], default=float) shard_counts[shard_idx] += 1 + if isinstance(item, dict): + total_turns += len(item.get("conversational_turns") or []) shard_paths = [] for idx in range(num_shards): @@ -261,8 +284,10 @@ def _stream_preshard(json_path: str, out_dir: str, num_shards: int) -> list[str] shard_paths.append(str(Path(out_dir) / f"shard_{idx}.json")) logger.info("Shard %d: %d samples", idx, shard_counts[idx]) - logger.info("Pre-sharding complete: %d total samples", sum(shard_counts)) - return shard_paths + logger.info( + "Pre-sharding complete: %d items, %d turns", sum(shard_counts), total_turns + ) + return shard_paths, total_turns def _resolve_test_json(task: str, json_filename: str, env: dict[str, str]) -> str: @@ -296,17 +321,28 @@ def _resolve_test_json(task: str, json_filename: str, env: dict[str, str]) -> st return str(Path(local_dir) / "raw" / json_filename) -def _aggregate_shards(shard_dir: str) -> dict[str, float]: +def _aggregate_shards( + shard_dir: str, *, expected_samples: int | None = None +) -> dict[str, float]: """Read per-shard output files and compute all metrics. - Each shard file contains a list of per-sample dicts with pre-computed - ``intersection``, ``union``, ``bbox_iou``, and ``round`` fields written - by the upstream ``evaluation_multi_segmentation.py`` script. + Each shard output file contains a list of per-turn dicts with + pre-computed ``image_id``, ``intersection``, ``union``, and ``bbox_iou`` + fields written by the upstream ``evaluation_multi_segmentation.py`` + script (there is no explicit round/turn field — rounds are inferred from + ``image_id`` occurrence order below). + + Output files are discovered recursively (``shard_*/output_0.json`` from + the per-shard run dirs, or flat ``output_*.json`` from single-GPU runs). Computes: - Aggregate metrics across all rounds: gIoU, cIoU, bbox_AP, pass_rate_* - Per-round metrics (R1–R7): gIoU_R1..R7, bbox_AP_R1..R7 + When *expected_samples* is given, raises if the aggregated sample count + differs — this catches shards that silently evaluated a partial (or + empty) slice, e.g. through upstream ``--idx``/``--num_parts`` drift. + Returns a flat dict of ``{metric_name: value}``. """ from oellm.contrib.regiondial_bench.metrics import ( @@ -316,7 +352,7 @@ def _aggregate_shards(shard_dir: str) -> dict[str, float]: PassRate, ) - shard_files = sorted(Path(shard_dir).glob("output_*.json")) + shard_files = sorted(Path(shard_dir).rglob("output_*.json")) if not shard_files: raise RuntimeError( f"No shard output files found in {shard_dir!r}. " @@ -334,6 +370,13 @@ def _aggregate_shards(shard_dir: str) -> dict[str, float]: "No samples found across shard files. " "The inference script produced empty output." ) + if expected_samples is not None and len(all_samples) != expected_samples: + raise RuntimeError( + f"Shard outputs contain {len(all_samples)} samples but the input " + f"had {expected_samples} turns. One or more shards evaluated a " + f"partial or empty slice — refusing to report metrics over a " + f"subset of the benchmark." + ) logger.info( "Aggregating %d samples from %d shards", len(all_samples), len(shard_files) ) diff --git a/oellm/envcheck.py b/oellm/envcheck.py new file mode 100644 index 00000000..cb7cc39a --- /dev/null +++ b/oellm/envcheck.py @@ -0,0 +1,464 @@ +"""Environment pre-flight: verify the runtime can actually execute the scheduled suites. + +The wrapper schedules work into *one* runtime per submission (a venv via +``--venv-path``, or the cluster's Singularity image), but the engines have +mutually incompatible dependency stacks (see ``docs/VENV.md``). Nothing else +in the pipeline validates that the chosen runtime contains the engines the +scheduled suites need — failures otherwise surface row-by-row on the compute +node hours later, or worse, run silently on the wrong engine version +(``dclm-core-22`` needs ``lm-eval==0.4.9.2``; a newer lm-eval *changes the +scores* instead of failing). + +Two entry points: + +* :func:`check_scheduled_environment` — called by the scheduler before + submission (bypass with ``--skip-checks``). Raises ``SystemExit`` listing + every problem at once. +* :func:`run_doctor_checks` — powers ``oellm-eval doctor``; returns a list of + :class:`CheckResult` for human-readable reporting. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +# Engines the cluster Singularity images contain (see containers/*.def: +# lm-eval on the system python, lighteval as an isolated uv tool). Everything +# else — lmms-eval, the oellm package itself (contrib dispatch), evalchemy — +# is NOT in the images and requires --venv-path. +_CONTAINER_SUPPORTED_SUITES = frozenset({"lm_eval", "lighteval"}) + +# Task groups that only produce valid numbers on a pinned engine version. +# Running them on any other version does not fail — it silently changes the +# scores, which is the worst possible failure for a benchmarking platform. +GROUP_VERSION_PINS: dict[str, tuple[tuple[str, str], ...]] = { + # v0.4.10+ breaks agieval_lsat_ar few-shot — see pyproject [dclm] extra. + "dclm-core-22": (("lm_eval", "0.4.9.2"),), +} + +_PROBE_TIMEOUT_S = 120 + + +@dataclass(frozen=True) +class SuiteRequirements: + """What a suite needs from the runtime to run at all.""" + + modules: tuple[str, ...] = () # importable by the venv's python + executables: tuple[str, ...] = () # in venv/bin or on PATH + env_vars: tuple[str, ...] = () # set in the scheduling environment + container_ok: bool = False + hint: str = "" + + +# Built-in engines. Contrib suites are resolved dynamically from the plugin +# registry (modules=("oellm",) + the plugin's declared CLUSTER_ENV_VARS). +SUITE_REQUIREMENTS: dict[str, SuiteRequirements] = { + "lm_eval": SuiteRequirements( + modules=("lm_eval",), + container_ok=True, + hint="install the [text] extra in the venv — see docs/VENV.md", + ), + "lighteval": SuiteRequirements( + executables=("lighteval",), + container_ok=True, + hint="install lighteval as a uv tool with UV_TOOL_BIN_DIR=/bin " + "— see docs/VENV.md", + ), + "lmms_eval": SuiteRequirements( + modules=("lmms_eval",), + container_ok=False, + hint="lmms-eval is not in the cluster containers; create the general " + "venv (docs/VENV.md) and pass --venv-path", + ), + "evalchemy": SuiteRequirements( + modules=("accelerate",), + env_vars=("EVALCHEMY_DIR",), + container_ok=False, + hint="evalchemy needs its own venv ([evalchemy] extra) and " + "EVALCHEMY_DIR pointing at the pinned clone — see docs/VENV.md", + ), +} + + +def _requirements_for_suite(canonical: str) -> SuiteRequirements | None: + """Look up requirements for a canonical suite name, contrib included.""" + if canonical in SUITE_REQUIREMENTS: + return SUITE_REQUIREMENTS[canonical] + + from oellm import registry + + try: + mod = registry.get_suite(canonical) + except KeyError: + return None + return SuiteRequirements( + modules=("oellm",), + env_vars=tuple(getattr(mod, "CLUSTER_ENV_VARS", ())), + container_ok=False, + hint=f"contrib suite — see oellm/contrib/{canonical}/README.md", + ) + + +def canonical_suites(suites: set[str] | list[str]) -> set[str]: + """Strip ``:model_flags`` suffixes and normalise engine aliases.""" + from oellm.runner import EvalRunner + + out = set() + for s in suites: + head = str(s).split(":", 1)[0].strip().lower() + if head: + out.add(EvalRunner.canonical_name(head)) + return out + + +def probe_import(python_bin: str | Path, module: str) -> tuple[bool, str]: + """Import *module* with *python_bin*; return (ok, version-or-error).""" + code = f"import {module} as _m; print(getattr(_m, '__version__', 'unknown version'))" + try: + r = subprocess.run( + [str(python_bin), "-c", code], + capture_output=True, + text=True, + timeout=_PROBE_TIMEOUT_S, + ) + except (OSError, subprocess.TimeoutExpired) as e: + return False, f"{type(e).__name__}: {e}" + if r.returncode == 0: + return True, r.stdout.strip() + err_lines = r.stderr.strip().splitlines() + return False, err_lines[-1] if err_lines else "import failed" + + +def _find_executable(name: str, venv_path: str | Path | None) -> str | None: + """Resolve *name* from the venv's bin dir first, then PATH.""" + if venv_path: + candidate = Path(venv_path).expanduser() / "bin" / name + if candidate.exists(): + return str(candidate) + return shutil.which(name) + + +def collect_problems( + suites: set[str] | list[str], + *, + venv_path: str | None, + group_names: list[str] | None = None, + env: dict | None = None, +) -> list[str]: + """Return a human-readable problem list for the scheduled suite set. + + ``venv_path=None`` means container mode (the cluster Singularity image), + which only supports lm-eval and lighteval. In venv mode each requirement + is probed against the venv's own interpreter, so the check reflects what + will actually run on the compute node. + """ + env = dict(os.environ) if env is None else env + problems: list[str] = [] + canonical = canonical_suites(suites) + venv_python = Path(venv_path).expanduser() / "bin" / "python" if venv_path else None + + for suite in sorted(canonical): + req = _requirements_for_suite(suite) + if req is None: + problems.append( + f"suite '{suite}': unknown — not a built-in engine and not a " + f"registered contrib plugin (the SLURM job would fail at " + f"dispatch time)" + ) + continue + + if venv_path is None: + if not req.container_ok: + problems.append( + f"suite '{suite}': not available in the cluster container " + f"image (it only ships lm-eval and lighteval). {req.hint}" + ) + # Container contents can't be probed cheaply from the login node; + # the static container_ok flag is the contract. + continue + + for module in req.modules: + ok, detail = probe_import(venv_python, module) + if not ok: + problems.append( + f"suite '{suite}': module '{module}' is not importable in " + f"venv {venv_path} ({detail}). {req.hint}" + ) + + for exe in req.executables: + if _find_executable(exe, venv_path) is None: + problems.append( + f"suite '{suite}': executable '{exe}' not found in " + f"{venv_path}/bin or on PATH. {req.hint}" + ) + + for var in req.env_vars: + value = env.get(var, "") + if not value: + problems.append( + f"suite '{suite}': required environment variable {var} is " + f"not set (add it to clusters.yaml for this cluster). " + f"{req.hint}" + ) + elif not Path(value).exists(): + problems.append( + f"suite '{suite}': {var}={value!r} does not exist on this " + f"filesystem. {req.hint}" + ) + + for group in group_names or []: + for module, required_version in GROUP_VERSION_PINS.get(group, ()): + if venv_path is None: + problems.append( + f"task group '{group}': requires {module}=={required_version}, " + f"but the container image ships an unpinned {module} — the " + f"run would produce silently wrong scores. Use the " + f"dedicated venv (docs/VENV.md)." + ) + continue + ok, version = probe_import(venv_python, module) + if ok and version != required_version: + problems.append( + f"task group '{group}': requires {module}=={required_version} " + f"but venv {venv_path} has {module}=={version} — scores " + f"would be silently wrong. Use the dedicated venv " + f"(docs/VENV.md)." + ) + + return problems + + +def check_scheduled_environment( + suites: set[str] | list[str], + *, + venv_path: str | None, + group_names: list[str] | None = None, + env: dict | None = None, +) -> None: + """Raise ``SystemExit`` if the runtime cannot execute the scheduled suites.""" + problems = collect_problems( + suites, venv_path=venv_path, group_names=group_names, env=env + ) + if problems: + bullet_list = "\n".join(f" - {p}" for p in problems) + raise SystemExit( + f"Environment pre-flight failed — the configured runtime cannot " + f"run the scheduled suites:\n{bullet_list}\n\n" + f"Fix the environment (docs/VENV.md), run `oellm-eval doctor` for " + f"a full report, or bypass with --skip-checks if you know better." + ) + + +# --------------------------------------------------------------------------- +# doctor +# --------------------------------------------------------------------------- + +OK = "ok" +WARN = "warn" +FAIL = "fail" + + +@dataclass +class CheckResult: + name: str + status: str # OK | WARN | FAIL + detail: str = "" + + +@dataclass +class _Doctor: + """Accumulates check results; every check is crash-isolated.""" + + results: list[CheckResult] = field(default_factory=list) + + def add(self, name: str, status: str, detail: str = "") -> None: + self.results.append(CheckResult(name, status, detail)) + + def run(self, name: str, fn) -> None: + try: + fn() + except Exception as e: # noqa: BLE001 — a broken check must not kill the report + self.add(name, WARN, f"check crashed: {type(e).__name__}: {e}") + + +def run_doctor_checks( + *, + venv_path: str | None = None, + task_groups: list[str] | None = None, +) -> list[CheckResult]: + """Run the full environment diagnostic and return the results. + + When *task_groups* is given, the suites they expand to are treated as + required: missing engines become FAIL instead of WARN. + """ + d = _Doctor() + + # ── cluster detection + required vars ──────────────────────────────── + def _cluster() -> None: + import socket + + from oellm.utils import _load_cluster_env + + hostname = socket.gethostname() + try: + _load_cluster_env() + d.add("cluster detection", OK, f"hostname {hostname} matched") + except ValueError: + d.add( + "cluster detection", + WARN, + f"hostname {hostname} matches no clusters.yaml entry " + f"(fine on a laptop; SLURM submission needs a match or " + f"explicit env vars)", + ) + except RuntimeError as e: + d.add("cluster detection", FAIL, str(e)) + + d.run("cluster detection", _cluster) + + def _required_vars() -> None: + for var in ( + "PARTITION", + "ACCOUNT", + "EVAL_BASE_DIR", + "EVAL_OUTPUT_DIR", + "GPUS_PER_NODE", + "HF_HOME", + ): + value = os.environ.get(var, "") + if value and "{" not in value: + d.add(f"env: {var}", OK, value) + else: + d.add( + f"env: {var}", + WARN, + "not set / unresolved (required for SLURM submission)", + ) + + d.run("required env vars", _required_vars) + + # ── HF cache ────────────────────────────────────────────────────────── + def _hf_home() -> None: + hf_home = os.environ.get("HF_HOME") + if not hf_home: + return # already reported above + p = Path(hf_home) + if not p.exists(): + d.add("HF_HOME path", WARN, f"{hf_home} does not exist yet") + return + if not os.access(p, os.W_OK): + d.add("HF_HOME path", FAIL, f"{hf_home} is not writable") + return + free_gb = shutil.disk_usage(p).free / 1e9 + status = OK if free_gb > 30 else WARN + d.add("HF_HOME path", status, f"{hf_home} (free: {free_gb:.0f} GB)") + + d.run("HF_HOME path", _hf_home) + + # ── SLURM binaries ──────────────────────────────────────────────────── + def _slurm() -> None: + for binary in ("sbatch", "squeue"): + path = shutil.which(binary) + d.add( + f"slurm: {binary}", + OK if path else WARN, + path or "not on PATH (only matters for SLURM submission)", + ) + + d.run("slurm binaries", _slurm) + + # ── container image ─────────────────────────────────────────────────── + def _container() -> None: + base = os.environ.get("EVAL_BASE_DIR", "") + image = os.environ.get("EVAL_CONTAINER_IMAGE", "") + if not (base and image): + d.add("container image", WARN, "EVAL_BASE_DIR/EVAL_CONTAINER_IMAGE not set") + return + path = Path(base) / image + if path.exists(): + import datetime + + age_days = ( + datetime.datetime.now() + - datetime.datetime.fromtimestamp(path.stat().st_mtime) + ).days + d.add("container image", OK, f"{path} ({age_days} days old)") + else: + d.add( + "container image", + WARN, + f"{path} not present (fetched automatically at schedule time)", + ) + + d.run("container image", _container) + + # ── which suites are required? ──────────────────────────────────────── + required_suites: set[str] = set() + if task_groups: + + def _expand() -> None: + from oellm.task_groups import _expand_task_groups + + expanded = _expand_task_groups(task_groups) + required_suites.update(canonical_suites({r.suite for r in expanded})) + d.add( + "task groups", + OK, + f"{', '.join(task_groups)} → suites: " + f"{', '.join(sorted(required_suites))}", + ) + + d.run("task groups", _expand) + + # ── venv engine probes ──────────────────────────────────────────────── + def _venv() -> None: + if not venv_path: + if required_suites - _CONTAINER_SUPPORTED_SUITES: + missing = sorted(required_suites - _CONTAINER_SUPPORTED_SUITES) + d.add( + "runtime mode", + FAIL, + f"no --venv-path, and the container image cannot run: " + f"{', '.join(missing)} (it only ships lm-eval + lighteval)", + ) + else: + d.add("runtime mode", OK, "container mode (lm-eval + lighteval)") + return + + venv = Path(venv_path).expanduser() + python_bin = venv / "bin" / "python" + if not python_bin.exists(): + d.add("venv", FAIL, f"{python_bin} does not exist") + return + d.add("venv", OK, str(venv)) + + probe_targets = set(SUITE_REQUIREMENTS) | required_suites + for suite in sorted(set(probe_targets)): + req = _requirements_for_suite(suite) + if req is None: + continue + needed = suite in required_suites + for module in req.modules: + ok, detail = probe_import(python_bin, module) + status = OK if ok else (FAIL if needed else WARN) + d.add(f"venv: import {module} ({suite})", status, detail) + for exe in req.executables: + found = _find_executable(exe, venv_path) + status = OK if found else (FAIL if needed else WARN) + d.add(f"venv: executable {exe} ({suite})", status, found or "not found") + for var in req.env_vars: + value = os.environ.get(var, "") + if value and Path(value).exists(): + d.add(f"env: {var} ({suite})", OK, value) + else: + status = FAIL if needed else WARN + detail = f"{value!r} does not exist" if value else "not set" + d.add(f"env: {var} ({suite})", status, detail) + + d.run("venv", _venv) + + return d.results diff --git a/oellm/main.py b/oellm/main.py index 29523516..893e7737 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -36,16 +36,16 @@ def schedule_evals( eval_csv_path: str | None = None, *, config: str | None = None, - max_array_len: int = 128, + max_array_len: int | None = None, limit: int | None = None, - verbose: bool = False, - download_only: bool = False, - dry_run: bool = False, - skip_checks: bool = False, - trust_remote_code: bool = True, + verbose: bool | None = None, + download_only: bool | None = None, + dry_run: bool | None = None, + skip_checks: bool | None = None, + trust_remote_code: bool | None = None, venv_path: str | None = None, lm_eval_include_path: str | None = None, - local: bool = False, + local: bool | None = None, slurm_template_var: str | None = None, allow_missing_judge: bool = False, ) -> None: @@ -53,8 +53,9 @@ def schedule_evals( Args: models: A string of comma-separated model paths or Hugging Face model identifiers. - Warning: does not allow passing model args such as `EleutherAI/pythia-160m,revision=step100000` - since we split on commas. If you need to pass model args, use the `eval_csv_path` option. + Warning: model args such as `EleutherAI/pythia-160m,revision=step100000` are + not supported — commas separate models here, and the SLURM-side CSV reader + also splits rows on commas, so `eval_csv_path` cannot carry them either. For local paths: - If a directory contains `.safetensors` files directly, it will be treated as a single model - If a directory contains subdirectories with models (e.g., converted_checkpoints/), @@ -68,14 +69,18 @@ def schedule_evals( n_shot: An integer or list of integers specifying the number of shots applied to `tasks`. eval_csv_path: A path to a CSV file containing evaluation data. Warning: exclusive argument. Cannot specify `models`, `tasks`, `task_groups`, or `n_shot` when `eval_csv_path` is provided. - config: Path to a YAML config file. CLI flags override YAML values. - max_array_len: The maximum number of jobs to schedule to run concurrently. + config: Path to a YAML config file. CLI flags override YAML values + (explicitly passing a flag wins even when it equals the default, + e.g. `--no-dry-run` overrides a YAML `dry_run: true`). + max_array_len: The maximum number of jobs to schedule to run concurrently. Default 128. Warning: this is not the number of jobs in the array job. This is determined by the environment variable `QUEUE_LIMIT`. limit: If set, limit the number of samples per task (useful for quick testing). Passes --limit to lm_eval and --max_samples to lighteval. download_only: If True, only download the datasets and models and exit. dry_run: If True, generate the SLURM script but don't submit it to the scheduler. - skip_checks: If True, skip container image, model validation, and dataset pre-download checks for faster execution. + skip_checks: If True, skip container image, environment pre-flight (engine availability + in the venv/container per scheduled suite), model validation, and dataset + pre-download checks for faster execution. trust_remote_code: If True, trust remote code when downloading datasets. Default is True. Workflow might fail if set to False. venv_path: Path to a Python virtual environment. If provided, evaluations run directly using this venv instead of inside a Singularity/Apptainer container. @@ -274,6 +279,56 @@ def _index(results: list[dict]) -> dict[tuple, float]: console.print(table) +def doctor( + *, + venv_path: str | None = None, + task_groups: str | None = None, +) -> None: + """Diagnose the evaluation environment. + + Checks cluster detection, required environment variables, HF cache health, + SLURM binaries, the container image, and — when --venv-path is given — + probes the venv for every engine the platform knows about (lm-eval, + lighteval, lmms-eval, evalchemy, contrib plugins). + + Args: + venv_path: Venv to probe; same value you would pass to schedule. + task_groups: Comma-separated task groups you intend to run. Engines + these groups need are treated as required — if they are missing, + the command exits non-zero. Without this, missing engines are + reported as warnings only. + """ + from rich.table import Table + + from oellm.envcheck import FAIL, OK, WARN, run_doctor_checks + from oellm.utils import get_console + + groups_list = ( + [g.strip() for g in task_groups.split(",") if g.strip()] if task_groups else None + ) + results = run_doctor_checks(venv_path=venv_path, task_groups=groups_list) + + console = get_console() + table = Table(title="oellm-eval doctor") + table.add_column("Check", style="bold") + table.add_column("Status") + table.add_column("Detail", overflow="fold") + icons = { + OK: "[green]OK[/green]", + WARN: "[yellow]WARN[/yellow]", + FAIL: "[red]FAIL[/red]", + } + for r in results: + table.add_row(r.name, icons.get(r.status, r.status), r.detail) + console.print(table) + + failures = [r for r in results if r.status == FAIL] + if failures: + console.print(f"[red]{len(failures)} check(s) failed.[/red]") + raise typer.Exit(1) + console.print("[green]No failing checks.[/green]") + + def eval_command( config: str | None = None, *, @@ -282,16 +337,16 @@ def eval_command( task_groups: str | None = None, n_shot: list[int] | None = None, eval_csv_path: str | None = None, - max_array_len: int = 128, + max_array_len: int | None = None, limit: int | None = None, - verbose: bool = False, - download_only: bool = False, - dry_run: bool = False, - skip_checks: bool = False, - trust_remote_code: bool = True, + verbose: bool | None = None, + download_only: bool | None = None, + dry_run: bool | None = None, + skip_checks: bool | None = None, + trust_remote_code: bool | None = None, venv_path: str | None = None, lm_eval_include_path: str | None = None, - local: bool = False, + local: bool | None = None, slurm_template_var: str | None = None, allow_missing_judge: bool = False, ) -> None: @@ -346,6 +401,7 @@ def eval_command( app.command("collect")(collect_results) app.command("list-tasks")(list_tasks) app.command("compare")(compare) +app.command("doctor")(doctor) def main() -> None: diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index e2fc0a44..d0f5ee89 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -65,9 +65,12 @@ fi # Use `tail` and `head` to slice the CSV file for the tasks assigned to this job. # The +1 on START_INDEX accounts for the header row. -tail -n +$((START_INDEX + 1)) "$CSV_PATH" | head -n $((END_INDEX - START_INDEX + 1)) | \ +# Process substitution (not a pipe) keeps the loop in the current shell so +# FAILED_ROWS survives past `done` and the job can exit non-zero on failures. +FAILED_ROWS=0 while IFS=, read -r model_path task_path n_shot eval_suite do + rc=0 # Strip CR (Windows line endings) from every field. model_path=${{model_path%$'\r'}} task_path=${{task_path%$'\r'}} @@ -153,6 +156,7 @@ do --trust_remote_code \ ${{LM_EVAL_INCLUDE_PATH:+--include_path $LM_EVAL_INCLUDE_PATH}} \ ${{LIMIT:+--limit $LIMIT}} + rc=$? echo "----------------------------------------------------" ;; lighteval|light-eval) @@ -198,6 +202,7 @@ do --output-dir "$RESULTS_SUBDIR" \ ${{LIMIT:+--max-samples $LIMIT}} fi + rc=$? ;; lmms_eval|lmms-eval) # lmms-eval is used for image/video/audio benchmarks. @@ -226,24 +231,29 @@ do --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ ${{LIMIT:+--limit $LIMIT}} + rc=$? ;; evalchemy) EVALCHEMY_WORK_DIR="{evalchemy_dir}" RESULTS_SUBDIR="{evals_dir}/$(openssl rand -hex 5)" if [ -n "$VENV_PATH" ]; then - source "$VENV_PATH/bin/activate" - cd "$EVALCHEMY_WORK_DIR" - accelerate launch --num-processes "$GPUS_PER_NODE" --num-machines 1 \ - $MULTI_GPU_FLAG -m eval.eval \ - --model hf \ - --tasks "$task_path" \ - --model_args "trust_remote_code=True,pretrained=$model_path" \ - --batch_size auto \ - --output_path "$RESULTS_SUBDIR" \ - ${{LIMIT:+--limit $LIMIT}} + # Subshell: contain the cd so later rows keep the original cwd. + ( + source "$VENV_PATH/bin/activate" + cd "$EVALCHEMY_WORK_DIR" || exit 1 + accelerate launch --num-processes "$GPUS_PER_NODE" --num-machines 1 \ + $MULTI_GPU_FLAG -m eval.eval \ + --model hf \ + --tasks "$task_path" \ + --model_args "trust_remote_code=True,pretrained=$model_path" \ + --batch_size auto \ + --output_path "$RESULTS_SUBDIR" \ + ${{LIMIT:+--limit $LIMIT}} + ) + rc=$? else echo "[error] evalchemy suite requires --venv-path (not supported in container mode)." - exit 1 + rc=1 fi ;; *) @@ -255,12 +265,23 @@ do --task "$task_path" \ --n_shot "$n_shot" \ --output_path "$_CONTRIB_OUTPUT" + rc=$? ;; esac echo "----------------------------------------------------" - echo "Evaluation finished for model: $model_path" + if [ "$rc" -ne 0 ]; then + FAILED_ROWS=$((FAILED_ROWS + 1)) + echo "[ERROR] Evaluation FAILED (exit=$rc) for model=$model_path task=$task_path n_shot=$n_shot suite=$eval_suite" + else + echo "Evaluation finished for model: $model_path" + fi -done +done < <(tail -n +$((START_INDEX + 1)) "$CSV_PATH" | head -n $((END_INDEX - START_INDEX + 1))) + +if [ "$FAILED_ROWS" -gt 0 ]; then + echo "Job $SLURM_ARRAY_TASK_ID finished with $FAILED_ROWS failed evaluation(s)." + exit 1 +fi echo "Job $SLURM_ARRAY_TASK_ID finished." diff --git a/oellm/results.py b/oellm/results.py index 37ce33fa..45b667eb 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -201,6 +201,22 @@ def _resolve_n_shot( return n_shot +def _model_paths_match(scheduled: str, completed: str) -> bool: + """Match a scheduled model_path against a model id found in a result JSON. + + Result JSONs often record a suffix of the scheduled path (a basename or an + HF id), so path-component-suffix matches are accepted in both directions. + Bare substring containment is NOT: it marks ``…/pythia-160m-deduped`` as + completed by a ``pythia-160m`` result and silently drops the job from the + missing list. + """ + if scheduled == completed: + return True + if scheduled.endswith("/" + completed) or completed.endswith("/" + scheduled): + return True + return False + + def _load_task_metrics() -> dict: """Load task_metrics from core YAML and all contrib suites.""" task_groups_yaml = files("oellm.resources") / "task-groups.yaml" @@ -288,8 +304,16 @@ def collect_results( completed_jobs = set() for json_file in json_files: - with open(json_file) as f: - data = json.load(f) + # A truncated file (OOM-killed / timed-out SLURM task) must not abort + # the whole collection; skip it and let --check report the job missing. + try: + with open(json_file) as f: + data = json.load(f) + except (json.JSONDecodeError, UnicodeDecodeError, OSError) as e: + logging.warning( + f"Skipping unreadable result file {json_file}: {type(e).__name__}: {e}" + ) + continue # Model name lives in different keys depending on the harness: # - lmms-eval: model_name_or_path is the checkpoint, model_name is the @@ -515,9 +539,8 @@ def collect_results( if ( job["n_shot"] == completed_n_shot and job["task_path"] == completed_task - and ( - str(job["model_path"]).endswith(completed_model) - or completed_model in str(job["model_path"]) + and _model_paths_match( + str(job["model_path"]), str(completed_model) ) ): is_completed = True @@ -534,7 +557,10 @@ def collect_results( if len(missing_jobs) > 0: missing_df = pd.DataFrame(missing_jobs) - missing_csv = output_csv.replace(".csv", "_missing.csv") + _out = Path(output_csv) + missing_csv = str( + _out.with_name(f"{_out.stem}_missing{_out.suffix or '.csv'}") + ) missing_df.to_csv(missing_csv, index=False) logging.info(f"Missing jobs saved to: {missing_csv}") logging.info( diff --git a/oellm/scheduler.py b/oellm/scheduler.py index c8c39064..e74e71c2 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -124,8 +124,9 @@ def schedule_evals( Args: models: A string of comma-separated model paths or Hugging Face model identifiers. - Warning: does not allow passing model args such as `EleutherAI/pythia-160m,revision=step100000` - since we split on commas. If you need to pass model args, use the `eval_csv_path` option. + Warning: model args such as `EleutherAI/pythia-160m,revision=step100000` are + not supported — commas separate models here, and the SLURM-side CSV reader + also splits rows on commas, so `eval_csv_path` cannot carry them either. For local paths: - If a directory contains `.safetensors` files directly, it will be treated as a single model - If a directory contains subdirectories with models (e.g., converted_checkpoints/), @@ -300,6 +301,18 @@ def schedule_evals( runner.prepare_jobs(expanded_eval_jobs) if not skip_checks: + # Verify the runtime can actually execute the scheduled suites before + # any network work: missing engines otherwise fail row-by-row on the + # compute node hours later, and a version-pinned group (dclm-core-22) + # on the wrong engine produces silently wrong scores. + from oellm.envcheck import check_scheduled_environment + + check_scheduled_environment( + {job.eval_suite for job in expanded_eval_jobs}, + venv_path=venv_path, + group_names=group_names, + ) + hub_models: set[str | Path] = { job.model_path for job in expanded_eval_jobs @@ -328,6 +341,18 @@ def _lower_suite_only(s: str) -> str: df["eval_suite"] = df["eval_suite"].map(_lower_suite_only) + # The SLURM-side reader slices the CSV with bash `IFS=, read`, which cannot + # parse quoted fields — a comma/quote/newline in any value would be split + # into the wrong columns silently at eval time. Refuse early instead. + for _col in ("model_path", "task_path", "eval_suite"): + _bad = df[df[_col].astype(str).str.contains(r'[,"\n\r]', regex=True)] + if not _bad.empty: + raise ValueError( + f"{_col} value {_bad.iloc[0][_col]!r} contains a comma, quote, or " + f"newline, which the SLURM job's CSV reader cannot parse. Model " + f"args like 'model,revision=...' are not supported." + ) + # Ensure that all datasets required by the tasks are cached locally to avoid # network access on compute nodes. if not skip_checks: @@ -401,7 +426,9 @@ def _lower_suite_only(s: str) -> str: sbatch_template = (files("oellm.resources") / "template.sbatch").read_text() total_evals = len(df) - actual_array_size = min(remaining_queue_capacity, total_evals) + # max(1, …): with --dry-run the zero-capacity early-return above is skipped, + # and a full queue would otherwise make this 0 (ZeroDivisionError below). + actual_array_size = max(1, min(remaining_queue_capacity, total_evals)) evals_per_job = max(1, int(math.ceil(total_evals / actual_array_size))) time_limit = os.environ.get("TIME_LIMIT", "12:00:00") @@ -501,6 +528,7 @@ def _lower_suite_only(s: str) -> str: logging.info("Local evaluation completed.") except subprocess.CalledProcessError as e: logging.error(f"Evaluation failed with exit code {e.returncode}") + raise SystemExit(e.returncode or 1) from e return try: @@ -526,7 +554,9 @@ def _lower_suite_only(s: str) -> str: except subprocess.CalledProcessError as e: logging.error(f"Failed to submit job: {e}") logging.error(f"sbatch stderr: {e.stderr}") - except FileNotFoundError: + raise SystemExit(1) from e + except FileNotFoundError as e: logging.error( "sbatch command not found. Please make sure you are on a system with SLURM installed." ) + raise SystemExit(1) from e diff --git a/oellm/task_groups.py b/oellm/task_groups.py index dd684ed9..d9e11839 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -4,6 +4,12 @@ import yaml +# Datasets that store media as external URLs (not embedded bytes). A bare +# snapshot_download only fetches the URLs, not the media — they MUST go through +# load_dataset()+_materialize_external_urls() so the per-row HTTP fetch runs on +# the (online) login node. Excluded from the snapshot-only fast path below. +_URL_BASED_DATASETS = {"facebook/textvqa"} + @dataclass class DatasetSpec: @@ -235,7 +241,11 @@ def add_spec( existing.needs_snapshot_download = True for t, _, group_name in _iter_all_tasks(parsed): - needs_snapshot = group_name.startswith(("audio-", "video-")) + needs_snapshot = group_name.startswith(("audio-", "video-", "image-")) + if t.dataset in _URL_BASED_DATASETS: + # URL-based dataset: snapshot would grab only links. Force the + # load_dataset()+materialize path so images are actually fetched. + needs_snapshot = False if t.dataset == "facebook/flores" and not t.subset: for lang in _extract_flores_subsets(t.name): diff --git a/oellm/utils.py b/oellm/utils.py index f9975778..5a656d7e 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -166,12 +166,16 @@ def __missing__(self, key): os.environ.setdefault(k, v) # Validate that critical sbatch variables resolved to real values. + # HF_HOME is included because the job script derives every cache path from + # it (`HF_DATASETS_CACHE="$HF_HOME/datasets"`); unset, the compute node + # would resolve caches to "/datasets" and fail far from the real cause. _required_vars = [ "PARTITION", "ACCOUNT", "EVAL_BASE_DIR", "EVAL_OUTPUT_DIR", "GPUS_PER_NODE", + "HF_HOME", ] missing = [ v for v in _required_vars if not os.environ.get(v) or "{" in os.environ.get(v, "") @@ -304,10 +308,12 @@ def _process_model_paths(models: Iterable[str]): ) per_model_paths.append(model) except Exception as e: - logging.debug( - f"Failed to download model {model} from Hugging Face Hub. Continuing..." + logging.warning( + f"Failed to download model {model} from Hugging Face Hub " + f"({type(e).__name__}: {e}). The job will still be " + f"scheduled and will fail on the offline compute node " + f"unless the model is already cached." ) - logging.debug(e) else: cache_dir = ( Path(os.getenv("HF_HOME")) / "hub" @@ -459,6 +465,18 @@ def _pre_download_datasets_from_specs( f"falling back to load_dataset." ) + if spec.needs_snapshot_download and not snapshot_failed: + # Snapshot already cached the raw files. Skip the + # load_dataset() build below: it fully materializes the + # dataset into Arrow (decoding every image/audio sample), + # which OOM-kills large media datasets on the memory-capped + # login node. The compute node builds at runtime from the + # cached files (offline-safe and has far more RAM). + logging.debug( + f"Snapshot complete for '{label}'; skipping load_dataset build." + ) + continue + try: ds = load_dataset( spec.repo_id, diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 90b790fc..2a19598b 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -476,3 +476,120 @@ def test_no_structured_output_when_no_results(self, tmp_path): assert not (tmp_path / "out.json").exists() assert not (tmp_path / "out.md").exists() + + +# ── corrupt result files must not abort collection ─────────────────────────── + + +class TestCorruptResultFiles: + def test_truncated_json_is_skipped(self, tmp_path): + results_dir = tmp_path / "results" + results_dir.mkdir() + write_result( + results_dir, + { + "model_name": "/path/to/model", + "results": {"mmlu": {"acc,none": 0.75}}, + "n-shot": {"mmlu": 5}, + }, + filename="good.json", + ) + # Simulate an OOM-killed job's torn write. + (results_dir / "truncated.json").write_text('{"model_name": "/path/to/mo') + + output_csv = str(tmp_path / "out.csv") + # NOTE: collect_results calls _setup_logging, which replaces root + # handlers — caplog can't observe the skip warning. The behavioral + # assertions below are the contract: the good file is collected, + # the truncated one doesn't abort the run. + collect_results(str(results_dir), output_csv=output_csv) + + df = pd.read_csv(output_csv) + assert len(df) == 1 + assert df.iloc[0]["performance"] == pytest.approx(0.75) + + def test_corrupt_json_counts_as_missing_in_check(self, tmp_path): + (tmp_path / "jobs.csv").write_text( + "model_path,task_path,n_shot,eval_suite\n/models/pythia-160m,mmlu,5,lm_eval\n" + ) + results_dir = tmp_path / "results" + results_dir.mkdir() + (results_dir / "truncated.json").write_text("[not json") + + output_csv = str(tmp_path / "out.csv") + collect_results(str(tmp_path), output_csv=output_csv, check=True) + + missing = pd.read_csv(tmp_path / "out_missing.csv") + assert len(missing) == 1 + assert missing.iloc[0]["model_path"] == "/models/pythia-160m" + + +# ── --check completion matching: exact/suffix, not substring ───────────────── + + +class TestCheckModeMatching: + def _run_check(self, tmp_path, jobs_rows: list[str], result_payloads: list[dict]): + (tmp_path / "jobs.csv").write_text( + "model_path,task_path,n_shot,eval_suite\n" + "".join(jobs_rows) + ) + results_dir = tmp_path / "results" + results_dir.mkdir() + for i, payload in enumerate(result_payloads): + write_result(results_dir, payload, filename=f"r{i}.json") + output_csv = str(tmp_path / "out.csv") + collect_results(str(tmp_path), output_csv=output_csv, check=True) + missing_path = tmp_path / "out_missing.csv" + if not missing_path.exists(): + return pd.DataFrame() + return pd.read_csv(missing_path) + + def test_shared_prefix_model_is_not_marked_complete(self, tmp_path): + """A pythia-160m result must NOT complete the pythia-160m-deduped job.""" + missing = self._run_check( + tmp_path, + [ + "/models/pythia-160m,mmlu,5,lm_eval\n", + "/models/pythia-160m-deduped,mmlu,5,lm_eval\n", + ], + [ + { + "model_name": "/models/pythia-160m", + "results": {"mmlu": {"acc,none": 0.75}}, + "n-shot": {"mmlu": 5}, + } + ], + ) + assert len(missing) == 1 + assert missing.iloc[0]["model_path"] == "/models/pythia-160m-deduped" + + def test_basename_result_completes_full_path_job(self, tmp_path): + """Result JSONs often record only a suffix of the scheduled path.""" + missing = self._run_check( + tmp_path, + ["/abs/checkpoints/pythia-160m,mmlu,5,lm_eval\n"], + [ + { + "model_name": "pythia-160m", + "results": {"mmlu": {"acc,none": 0.75}}, + "n-shot": {"mmlu": 5}, + } + ], + ) + assert len(missing) == 0 + + def test_missing_csv_name_without_suffix(self, tmp_path): + """An output name without .csv must not be overwritten by the missing list.""" + (tmp_path / "jobs.csv").write_text( + "model_path,task_path,n_shot,eval_suite\n/models/pythia-160m,mmlu,5,lm_eval\n" + ) + results_dir = tmp_path / "results" + results_dir.mkdir() + + output_csv = str(tmp_path / "outfile") + collect_results(str(tmp_path), output_csv=output_csv, check=True) + + assert (tmp_path / "outfile_missing.csv").exists() + assert not (tmp_path / "outfile").exists() or ( + (tmp_path / "outfile").read_text() + != (tmp_path / "outfile_missing.csv").read_text() + ) diff --git a/tests/test_envcheck.py b/tests/test_envcheck.py new file mode 100644 index 00000000..94508898 --- /dev/null +++ b/tests/test_envcheck.py @@ -0,0 +1,277 @@ +"""Tests for the environment pre-flight (oellm/envcheck.py) and doctor.""" + +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +from oellm.envcheck import ( + FAIL, + OK, + WARN, + SuiteRequirements, + canonical_suites, + check_scheduled_environment, + collect_problems, + probe_import, + run_doctor_checks, +) + +TEST_VENV = str(Path(sys.prefix)) + + +# --------------------------------------------------------------------------- +# probe_import +# --------------------------------------------------------------------------- + + +class TestProbeImport: + def test_stdlib_module_imports(self): + ok, detail = probe_import(Path(sys.prefix) / "bin" / "python", "json") + assert ok + assert detail # version string or "unknown version" + + def test_missing_module_fails_with_error_text(self): + ok, detail = probe_import( + Path(sys.prefix) / "bin" / "python", "definitely_not_a_module_xyz" + ) + assert not ok + assert "definitely_not_a_module_xyz" in detail + + def test_missing_interpreter_fails(self): + ok, detail = probe_import("/nonexistent/python", "json") + assert not ok + assert detail + + +# --------------------------------------------------------------------------- +# canonical_suites +# --------------------------------------------------------------------------- + + +class TestCanonicalSuites: + def test_strips_model_flags_and_normalises_aliases(self): + suites = { + "lm-eval-harness", + "lmms_eval:llava_hf", + "audiobench:Qwen2-Audio-7B-Instruct", + "LIGHTEVAL", + } + assert canonical_suites(suites) == { + "lm_eval", + "lmms_eval", + "audiobench", + "lighteval", + } + + +# --------------------------------------------------------------------------- +# collect_problems — container mode (static contract) +# --------------------------------------------------------------------------- + + +class TestContainerMode: + def test_lm_eval_and_lighteval_are_fine(self): + assert collect_problems({"lm_eval", "lighteval"}, venv_path=None) == [] + + def test_lmms_eval_is_rejected(self): + problems = collect_problems({"lmms_eval:llava_hf"}, venv_path=None) + assert len(problems) == 1 + assert "container" in problems[0] + + def test_contrib_suite_is_rejected(self): + problems = collect_problems( + {"audiobench:Qwen2-Audio-7B-Instruct"}, venv_path=None + ) + assert len(problems) == 1 + assert "container" in problems[0] + + def test_unknown_suite_is_rejected(self): + problems = collect_problems({"no_such_suite"}, venv_path=None) + assert len(problems) == 1 + assert "unknown" in problems[0] + + def test_pinned_group_is_rejected_in_container_mode(self): + problems = collect_problems( + {"lm_eval"}, venv_path=None, group_names=["dclm-core-22"] + ) + assert any("dclm-core-22" in p and "silently wrong" in p for p in problems) + + +# --------------------------------------------------------------------------- +# collect_problems — venv mode (live probes against this test venv) +# --------------------------------------------------------------------------- + + +class TestVenvMode: + def test_missing_engine_is_reported(self): + # The dev/test venv has no lm-eval installed — the probe must fail. + problems = collect_problems({"lm_eval"}, venv_path=TEST_VENV) + assert len(problems) == 1 + assert "lm_eval" in problems[0] + assert TEST_VENV in problems[0] + + def test_satisfied_requirement_passes(self): + fake = {"lm_eval": SuiteRequirements(modules=("json",), container_ok=True)} + with patch.dict("oellm.envcheck.SUITE_REQUIREMENTS", fake): + assert collect_problems({"lm_eval"}, venv_path=TEST_VENV) == [] + + def test_contrib_env_var_missing(self): + # oellm itself is importable in the test venv, so only the env var fails. + env = {k: v for k, v in os.environ.items() if k != "AUDIOBENCH_DIR"} + problems = collect_problems({"audiobench"}, venv_path=TEST_VENV, env=env) + assert len(problems) == 1 + assert "AUDIOBENCH_DIR" in problems[0] + assert "not set" in problems[0] + + def test_contrib_env_var_nonexistent_path(self, tmp_path): + env = dict(os.environ) + env["AUDIOBENCH_DIR"] = str(tmp_path / "missing") + problems = collect_problems({"audiobench"}, venv_path=TEST_VENV, env=env) + assert len(problems) == 1 + assert "does not exist" in problems[0] + + def test_contrib_fully_satisfied(self, tmp_path): + env = dict(os.environ) + env["AUDIOBENCH_DIR"] = str(tmp_path) + assert collect_problems({"audiobench"}, venv_path=TEST_VENV, env=env) == [] + + def test_version_pin_mismatch_reported(self): + # json imports fine but its "version" is never 9.9.9. + pins = {"some-group": (("json", "9.9.9"),)} + with patch.dict("oellm.envcheck.GROUP_VERSION_PINS", pins): + problems = collect_problems( + set(), venv_path=TEST_VENV, group_names=["some-group"] + ) + assert len(problems) == 1 + assert "9.9.9" in problems[0] + assert "silently wrong" in problems[0] + + def test_missing_executable_reported(self): + fake = { + "lighteval": SuiteRequirements( + executables=("definitely-not-a-binary-xyz",), container_ok=True + ) + } + with patch.dict("oellm.envcheck.SUITE_REQUIREMENTS", fake): + problems = collect_problems({"lighteval"}, venv_path=TEST_VENV) + assert len(problems) == 1 + assert "definitely-not-a-binary-xyz" in problems[0] + + +# --------------------------------------------------------------------------- +# check_scheduled_environment + scheduler wiring +# --------------------------------------------------------------------------- + + +class TestScheduledEnvironmentCheck: + def test_raises_systemexit_listing_all_problems(self): + with pytest.raises(SystemExit) as excinfo: + check_scheduled_environment( + {"lmms_eval:llava_hf", "no_such_suite"}, venv_path=None + ) + msg = str(excinfo.value) + assert "lmms_eval" in msg + assert "no_such_suite" in msg + assert "--skip-checks" in msg + + def test_scheduler_runs_the_check_when_not_skipping(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + with pytest.raises(SystemExit, match="lm_eval"): + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + venv_path=TEST_VENV, # dev venv has no lm-eval + dry_run=True, + ) + + def test_scheduler_passes_with_satisfied_requirements(self, tmp_path): + from oellm.main import schedule_evals + + fake = {"lm_eval": SuiteRequirements(modules=("json",), container_ok=True)} + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._process_model_paths"), + patch("oellm.scheduler._lookup_dataset_specs_for_tasks", return_value=[]), + patch.dict("oellm.envcheck.SUITE_REQUIREMENTS", fake), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + venv_path=TEST_VENV, + dry_run=True, + ) + assert list(tmp_path.glob("**/submit_evals.sbatch")) + + def test_skip_checks_bypasses_the_check(self, tmp_path): + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + venv_path=TEST_VENV, + dry_run=True, + skip_checks=True, + ) + assert list(tmp_path.glob("**/submit_evals.sbatch")) + + +# --------------------------------------------------------------------------- +# doctor +# --------------------------------------------------------------------------- + + +class TestDoctor: + def test_returns_structured_results(self): + results = run_doctor_checks(venv_path=TEST_VENV) + assert results + assert all(r.status in (OK, WARN, FAIL) for r in results) + names = {r.name for r in results} + assert "venv" in names + assert any(n.startswith("env: HF_HOME") or n == "HF_HOME path" for n in names) + + def test_venv_engine_probes_present(self): + results = run_doctor_checks(venv_path=TEST_VENV) + probe_names = {r.name for r in results if r.name.startswith("venv: import")} + # All built-in module-based engines must be probed. + assert any("lm_eval" in n for n in probe_names) + assert any("lmms_eval" in n for n in probe_names) + + def test_required_group_without_venv_fails_runtime_mode(self): + results = run_doctor_checks(venv_path=None, task_groups=["image-vqa"]) + runtime = [r for r in results if r.name == "runtime mode"] + assert runtime and runtime[0].status == FAIL + assert "lmms_eval" in runtime[0].detail + + def test_missing_engines_fail_when_group_requires_them(self): + results = run_doctor_checks(venv_path=TEST_VENV, task_groups=["image-vqa"]) + lmms_probes = [r for r in results if r.name.startswith("venv: import lmms_eval")] + assert lmms_probes and lmms_probes[0].status == FAIL + + def test_missing_engines_warn_when_not_required(self): + results = run_doctor_checks(venv_path=TEST_VENV) + lmms_probes = [r for r in results if r.name.startswith("venv: import lmms_eval")] + assert lmms_probes and lmms_probes[0].status == WARN + + def test_broken_venv_path_fails(self, tmp_path): + results = run_doctor_checks(venv_path=str(tmp_path / "nope")) + venv_checks = [r for r in results if r.name == "venv"] + assert venv_checks and venv_checks[0].status == FAIL diff --git a/tests/test_eval_config.py b/tests/test_eval_config.py index 67db125e..11f57b92 100644 --- a/tests/test_eval_config.py +++ b/tests/test_eval_config.py @@ -398,3 +398,106 @@ def test_ensure_model_list_missing_path_key(self, tmp_path): ) with pytest.raises(KeyError): EvalConfig.from_yaml(str(cfg_file)) + + +# --------------------------------------------------------------------------- +# merge provenance: explicit CLI values override YAML even at the default +# --------------------------------------------------------------------------- + + +class TestMergeProvenance: + def _yaml_cfg(self, **overrides): + raw = {"models": ["m"], "task_groups": ["g"], **overrides} + return EvalConfig._from_dict(raw) + + def test_no_dry_run_overrides_yaml_dry_run(self): + yaml_cfg = self._yaml_cfg(dry_run=True) + cli_cfg = EvalConfig.from_cli_kwargs(dry_run=False) + assert yaml_cfg.merge(cli_cfg).dry_run is False + + def test_trust_remote_code_overrides_yaml_false(self): + yaml_cfg = self._yaml_cfg(trust_remote_code=False) + cli_cfg = EvalConfig.from_cli_kwargs(trust_remote_code=True) + assert yaml_cfg.merge(cli_cfg).trust_remote_code is True + + def test_unprovided_bool_keeps_yaml_value(self): + yaml_cfg = self._yaml_cfg(dry_run=True, trust_remote_code=False) + cli_cfg = EvalConfig.from_cli_kwargs() + merged = yaml_cfg.merge(cli_cfg) + assert merged.dry_run is True + assert merged.trust_remote_code is False + + def test_default_valued_max_array_len_overrides_yaml(self): + yaml_cfg = self._yaml_cfg(slurm={"max_array_len": 64}) + cli_cfg = EvalConfig.from_cli_kwargs(max_array_len=128) + assert yaml_cfg.merge(cli_cfg).slurm.max_array_len == 128 + + def test_unprovided_max_array_len_keeps_yaml(self): + yaml_cfg = self._yaml_cfg(slurm={"max_array_len": 64}) + cli_cfg = EvalConfig.from_cli_kwargs() + assert yaml_cfg.merge(cli_cfg).slurm.max_array_len == 64 + + +# --------------------------------------------------------------------------- +# slurm_template_var passthrough: unmodeled keys must survive the round-trip +# --------------------------------------------------------------------------- + + +class TestSlurmTemplateVarPassthrough: + def test_slurm_mem_round_trips(self): + cfg = EvalConfig.from_cli_kwargs( + slurm_template_var='{"SLURM_MEM":"123G","PARTITION":"dev-g"}' + ) + assert cfg.slurm.extra_template_vars == {"SLURM_MEM": "123G"} + assert cfg.slurm.partition == "dev-g" + + rendered = cfg.slurm_template_var_json + assert rendered is not None + assert '"SLURM_MEM": "123G"' in rendered or '"SLURM_MEM":"123G"' in rendered + + def test_extras_survive_merge(self): + yaml_cfg = EvalConfig._from_dict({"models": ["m"], "task_groups": ["g"]}) + cli_cfg = EvalConfig.from_cli_kwargs( + slurm_template_var='{"SLURM_MEM":"96G","CPUS_PER_TASK":"8"}' + ) + merged = yaml_cfg.merge(cli_cfg) + assert merged.slurm.extra_template_vars == { + "SLURM_MEM": "96G", + "CPUS_PER_TASK": "8", + } + + def test_known_keys_not_duplicated_into_extras(self): + cfg = EvalConfig.from_cli_kwargs( + slurm_template_var='{"PARTITION":"p","ACCOUNT":"a","GPUS_PER_NODE":2,"TIME":"01:00:00"}' + ) + assert cfg.slurm.extra_template_vars == {} + d = cfg.slurm.to_template_var_dict() + assert d == { + "PARTITION": "p", + "ACCOUNT": "a", + "GPUS_PER_NODE": "2", + "TIME": "01:00:00", + } + + +# --------------------------------------------------------------------------- +# unknown YAML keys warn instead of vanishing silently +# --------------------------------------------------------------------------- + + +class TestUnknownYamlKeys: + def test_unknown_top_level_key_warns(self, tmp_path, caplog): + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text("models:\n - 'm'\ntask_groups:\n - 'g'\nn_shots: 5\n") + with caplog.at_level("WARNING"): + EvalConfig.from_yaml(str(cfg_file)) + assert "n_shots" in caplog.text + + def test_unknown_slurm_key_warns(self, tmp_path, caplog): + cfg_file = tmp_path / "eval.yaml" + cfg_file.write_text( + "models:\n - 'm'\ntask_groups:\n - 'g'\nslurm:\n partion: 'x'\n" + ) + with caplog.at_level("WARNING"): + EvalConfig.from_yaml(str(cfg_file)) + assert "partion" in caplog.text diff --git a/tests/test_judge_preflight.py b/tests/test_judge_preflight.py new file mode 100644 index 00000000..84aab959 --- /dev/null +++ b/tests/test_judge_preflight.py @@ -0,0 +1,73 @@ +"""Tests for the OPENAI_API_KEY pre-flight check in ``check_judge_llm_pre_flight``.""" + +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest + +from oellm.constants import JUDGE_REQUIRED_TASKS +from oellm.utils import check_judge_llm_pre_flight + + +def test_passes_when_no_judge_required_task() -> None: + """Non-judge tasks must not trigger the check regardless of API key state.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPENAI_API_KEY", None) + check_judge_llm_pre_flight(["vqav2_val", "mmmu_val", "chartqa"]) + + +def test_passes_when_openai_api_key_set() -> None: + """A set ``OPENAI_API_KEY`` (any non-empty value) satisfies the check.""" + with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-test"}): + check_judge_llm_pre_flight(["activitynetqa", "alpaca_audio"]) + + +def test_passes_with_allow_missing_judge() -> None: + """``allow_missing=True`` lets judge-required tasks through with a warning.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPENAI_API_KEY", None) + check_judge_llm_pre_flight(["activitynetqa", "alpaca_audio"], allow_missing=True) + + +def test_refuses_when_judge_required_and_no_key() -> None: + """Default-strict: judge-required task without key must abort.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPENAI_API_KEY", None) + with pytest.raises(SystemExit) as exc: + check_judge_llm_pre_flight(["activitynetqa", "vqav2_val"]) + # Error message must name the offending task and OPENAI_API_KEY. + assert "activitynetqa" in str(exc.value) + assert "OPENAI_API_KEY" in str(exc.value) + assert "allow-missing-judge" in str(exc.value) + + +def test_refuses_only_lists_judge_required_tasks() -> None: + """The error message must list ONLY the judge-required tasks, not all tasks.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPENAI_API_KEY", None) + with pytest.raises(SystemExit) as exc: + check_judge_llm_pre_flight( + ["vqav2_val", "alpaca_audio", "chartqa", "wavcaps"] + ) + msg = str(exc.value) + # Judge-required: present + assert "alpaca_audio" in msg + assert "wavcaps" in msg + # Non-judge: NOT present in the offending list + assert "vqav2_val" not in msg + assert "chartqa" not in msg + + +def test_empty_task_list_passes() -> None: + """No tasks → nothing to check.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("OPENAI_API_KEY", None) + check_judge_llm_pre_flight([]) + + +def test_judge_required_tasks_set_is_non_empty() -> None: + """Guard against accidentally emptying the set.""" + assert len(JUDGE_REQUIRED_TASKS) > 0 + assert "activitynetqa" in JUDGE_REQUIRED_TASKS diff --git a/tests/test_metric_snapshots.py b/tests/test_metric_snapshots.py new file mode 100644 index 00000000..1e1c889b --- /dev/null +++ b/tests/test_metric_snapshots.py @@ -0,0 +1,205 @@ +"""Tier 1 metric-resolution snapshot test. + +For every image and video benchmark wired in ``task-groups.yaml``'s +``task_metrics`` mapping, this test asserts that ``_resolve_metric`` returns a +non-null float when handed a realistic lmms-eval result_dict. + +The fixtures here are not real evaluation runs — they are minimal snippets that +mirror the JSON shape lmms-eval writes (``"/,none": ``). +The values are placeholders; what we are pinning is the **metric-key contract** +between this repo's YAML and lmms-eval's task definitions. If lmms-eval renames +a key upstream, this test fails immediately rather than letting a production +run silently emit ``null``. + +When adding a new benchmark to ``task_metrics``: + 1. Add the corresponding entry in ``SNAPSHOTS`` below. + 2. The metric-key in the fixture must match the value in ``task_metrics``. +""" + +from importlib.resources import files + +import pytest +import yaml + +from oellm.results import _resolve_metric + +# Realistic lmms-eval result_dict snippets (one per benchmark). +# Format mirrors what lmms-eval writes: ``"/,none": ``. +# Values are illustrative — the test only checks that the configured metric key +# resolves to a non-null float, not that any specific number is correct. +SNAPSHOTS: dict[str, dict] = { + # ── Image benchmarks ── + "vqav2_val": { + "vqav2_val/exact_match,none": 0.755, + "vqav2_val/exact_match_stderr,none": 0.004, + }, + "mmbench_en_dev": { + "mmbench_en_dev/gpt_eval_score,none": 72.4, + }, + "mmmu_val": { + "mmmu_val/mmmu_acc,none": 0.412, + }, + "chartqa": { + "chartqa/relaxed_overall,none": 0.681, + "chartqa/relaxed_human_split,none": 0.40, + "chartqa/relaxed_augmented_split,none": 0.96, + }, + "docvqa_val": { + "docvqa_val/anls,none": 0.832, + }, + "textvqa_val": { + "textvqa_val/exact_match,none": 0.604, + }, + # OCRBench: lmms-eval normalizes raw /1000 score into 0–1. + "ocrbench": { + "ocrbench/ocrbench_accuracy,none": 0.612, + }, + "mathvista_testmini_cot": { + "mathvista_testmini_cot/llm_as_judge_eval,none": 47.3, + }, + "mathvista_testmini_format": { + "mathvista_testmini_format/llm_as_judge_eval,none": 47.5, + }, + "mathvista_testmini_solution": { + "mathvista_testmini_solution/llm_as_judge_eval,none": 47.0, + }, + # ── Video benchmarks ── + "video_mmmu_perception": { + "video_mmmu_perception/mmmu_acc,none": 0.589, + }, + "video_mmmu_comprehension": { + "video_mmmu_comprehension/mmmu_acc,none": 0.512, + }, + "video_mmmu_adaptation": { + "video_mmmu_adaptation/mmmu_acc,none": 0.443, + }, + # MVBench: 0–100 from utils.mvbench_aggregate_results. + "mvbench": { + "mvbench/mvbench_accuracy,none": 56.2, + }, + # EgoSchema subset: 0–1, generation-style scoring on the 500-q val split. + "egoschema_subset": { + "egoschema_subset/score,none": 0.658, + }, + # VideoMME: 0–100. The "perception_score" name is upstream's misnomer for + # the overall accuracy without subtitles. + "videomme": { + "videomme/videomme_perception_score,none": 65.4, + }, + "activitynetqa": { + "activitynetqa/gpt_eval_accuracy,none": 51.7, + "activitynetqa/gpt_eval_score,none": 3.4, + }, + "longvideobench_val_v": { + "longvideobench_val_v/lvb_acc,none": 0.473, + }, +} + + +@pytest.fixture(scope="module") +def task_metrics() -> dict: + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + return data["task_metrics"] + + +@pytest.mark.parametrize("task_name", sorted(SNAPSHOTS.keys())) +def test_configured_metric_key_resolves_against_snapshot( + task_name: str, task_metrics: dict +) -> None: + """``_resolve_metric`` must return a non-null float for every wired benchmark. + + Failure mode this catches: lmms-eval renames a metric key (e.g. + ``mmmu_acc`` → ``accuracy``) and our YAML mapping silently produces ``null`` + in production results. + """ + expected_key = task_metrics.get(task_name) + assert expected_key is not None, ( + f"{task_name} has a snapshot but no entry in task-groups.yaml::task_metrics" + ) + + result_dict = SNAPSHOTS[task_name] + value, resolved_key = _resolve_metric(task_name, result_dict, task_metrics) + + assert value is not None, ( + f"_resolve_metric returned None for {task_name}. The fixture contains " + f"'{expected_key},none' but the configured mapping " + f"'{task_name}: {expected_key}' did not resolve. Either lmms-eval " + f"renamed the key upstream or the fixture is stale." + ) + assert isinstance(value, float) + assert resolved_key is not None + assert expected_key in resolved_key, ( + f"_resolve_metric for {task_name} resolved to '{resolved_key}', " + f"which does not contain configured key '{expected_key}'." + ) + + +def test_every_snapshot_has_a_task_metrics_entry(task_metrics: dict) -> None: + """Snapshots and task_metrics must be kept in lockstep — adding a new + benchmark requires both.""" + missing = [t for t in SNAPSHOTS if t not in task_metrics] + assert not missing, ( + f"Tasks have snapshots but no task_metrics entry: {missing}. " + f"Add them to task-groups.yaml::task_metrics or remove the snapshots." + ) + + +# List the image+video tasks that MUST have a snapshot. Sourced from the +# image-vqa, video-understanding task groups. +REQUIRED_TASKS_WITH_SNAPSHOT: set[str] = { + "vqav2_val", + "mmbench_en_dev", + "mmmu_val", + "chartqa", + "docvqa_val", + "textvqa_val", + "ocrbench", + "mathvista_testmini_cot", + "mathvista_testmini_format", + "mathvista_testmini_solution", + "video_mmmu_perception", + "video_mmmu_comprehension", + "video_mmmu_adaptation", + "mvbench", + "egoschema_subset", + "videomme", + "activitynetqa", + "longvideobench_val_v", +} + + +def test_all_required_image_video_tasks_have_snapshot() -> None: + """When a new image/video benchmark lands in REQUIRED_TASKS_WITH_SNAPSHOT, + its snapshot must land at the same time.""" + missing = REQUIRED_TASKS_WITH_SNAPSHOT - set(SNAPSHOTS.keys()) + assert not missing, ( + f"Required image/video tasks missing a snapshot fixture: {missing}. " + f"Add an entry in SNAPSHOTS for each before merging." + ) + + +# ── Normalization (post-resolve) ───────────────────────────────────────────── +# +# For every image+video benchmark, after _resolve_metric returns the raw +# value, the normalization helper must produce a 0–100 number. Catches the +# case where someone wires a new metric in task_metrics but forgets to +# register its native scale in METRIC_NATIVE_SCALE. + + +@pytest.mark.parametrize("task_name", sorted(SNAPSHOTS.keys())) +def test_normalized_value_is_in_zero_to_hundred_range( + task_name: str, task_metrics: dict +) -> None: + from oellm.results import _normalize_to_100 + + value, resolved_key = _resolve_metric(task_name, SNAPSHOTS[task_name], task_metrics) + normalized = _normalize_to_100(value, resolved_key) + assert normalized is not None, ( + f"_normalize_to_100 returned None for {task_name} " + f"(value={value}, key={resolved_key}). The metric's native scale is " + f"not registered in METRIC_NATIVE_SCALE in oellm/results.py." + ) + assert 0.0 <= normalized <= 100.0, ( + f"Normalized {task_name} = {normalized} is outside [0, 100]. " + f"Native scale entry in METRIC_NATIVE_SCALE is likely wrong." + ) diff --git a/tests/test_regiondial_bench.py b/tests/test_regiondial_bench.py index 54662d98..a6768920 100644 --- a/tests/test_regiondial_bench.py +++ b/tests/test_regiondial_bench.py @@ -789,3 +789,164 @@ def test_collect_results_parses_refcocoplus_json(self, tmp_path): row = df.iloc[0] assert row["task"] == RD_TASK_REFCOCOPLUS assert float(row["performance"]) == pytest.approx(0.55) + + +# --------------------------------------------------------------------------- +# Multi-GPU sharding (regression: shards 1..G-1 must not evaluate empty slices) +# --------------------------------------------------------------------------- + + +def _conversation(image_id: str, ious: list[tuple[int, int, float]]) -> dict: + """Build a test-JSON item with one conversational turn per (i, u, bbox) tuple.""" + return { + "image_id": image_id, + "image_path": f"{image_id}.jpg", + "conversational_turns": [ + {"question": f"q{k}", "bboxes": [[0, 0, 1, 1]], "ann_ids": [k]} + for k, _ in enumerate(ious) + ], + # carried separately for the fake inference below + "_expected_ious": ious, + } + + +class _FakeInferencePopen: + """Stands in for the RegionReasoner inference subprocess. + + Reads the shard file given via --test_data_path and writes one per-turn + record per conversational turn into --output_path/output_0.json — i.e. + behaves like the real script invoked correctly. Records every cmd for + later assertions. + """ + + calls: list[list[str]] = [] + returncode_for_dir: dict[str, int] = {} + write_empty_for_dir: set[str] = set() + + def __init__(self, cmd, env=None, cwd=None): + type(self).calls.append(list(cmd)) + out_dir = Path(cmd[cmd.index("--output_path") + 1]) + shard_file = Path(cmd[cmd.index("--test_data_path") + 1]) + self._rc = type(self).returncode_for_dir.get(out_dir.name, 0) + + records = [] + if out_dir.name not in type(self).write_empty_for_dir: + items = json.loads(shard_file.read_text()) + for item in items: + for inter, union, bbox_iou in item["_expected_ious"]: + records.append( + { + "image_id": str(item["image_id"]), + "intersection": inter, + "union": union, + "bbox_iou": bbox_iou, + } + ) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "output_0.json").write_text(json.dumps(records)) + + def wait(self): + return self._rc + + +class TestMultiGpuSharding: + @pytest.fixture + def rr_env(self, tmp_path): + """A fake REGION_REASONER_DIR plus a 2-conversation test JSON.""" + rr_dir = tmp_path / "RegionReasoner" + script = rr_dir / "test" / "evaluation" / "evaluation_multi_segmentation.py" + script.parent.mkdir(parents=True) + script.write_text("# fake inference script\n") + + test_json = tmp_path / "refcocog_multi_turn.json" + test_json.write_text( + json.dumps( + [ + _conversation("img1", [(100, 100, 1.0), (0, 100, 0.0)]), + _conversation("img2", [(50, 100, 0.6), (80, 100, 0.8)]), + ] + ) + ) + return { + "REGION_REASONER_DIR": str(rr_dir), + "GPUS_PER_NODE": "2", + "REGION_REASONER_TEST_JSON_REFCOCOG": str(test_json), + } + + @pytest.fixture(autouse=True) + def _reset_fake(self): + _FakeInferencePopen.calls = [] + _FakeInferencePopen.returncode_for_dir = {} + _FakeInferencePopen.write_empty_for_dir = set() + yield + + def _run(self, rr_env, tmp_path): + from oellm.contrib.regiondial_bench import suite + + output_path = tmp_path / "result.json" + with patch.object(suite.subprocess, "Popen", _FakeInferencePopen): + suite.run( + model_path="/models/RegionReasoner-7B", + task=RD_TASK_REFCOCOG, + n_shot=0, + output_path=output_path, + model_flags="vision_reasoner", + env=rr_env, + ) + return output_path + + def test_stream_preshard_counts_turns(self, tmp_path): + from oellm.contrib.regiondial_bench.suite import _stream_preshard + + src = tmp_path / "src.json" + src.write_text( + json.dumps( + [ + _conversation("a", [(1, 1, 1.0), (1, 1, 1.0)]), + _conversation("b", [(1, 1, 1.0)]), + _conversation("c", [(1, 1, 1.0)] * 3), + ] + ) + ) + out = tmp_path / "shards" + out.mkdir() + paths, total_turns = _stream_preshard(str(src), str(out), 2) + assert len(paths) == 2 + assert total_turns == 6 + # conversations stay whole: 2 items in shard 0, 1 in shard 1 + assert len(json.loads(Path(paths[0]).read_text())) == 2 + assert len(json.loads(Path(paths[1]).read_text())) == 1 + + def test_every_shard_runs_with_idx_zero_and_own_output_dir(self, rr_env, tmp_path): + self._run(rr_env, tmp_path) + + assert len(_FakeInferencePopen.calls) == 2 + out_dirs = set() + for cmd in _FakeInferencePopen.calls: + assert cmd[cmd.index("--idx") + 1] == "0" + assert cmd[cmd.index("--num_parts") + 1] == "1" + out_dirs.add(cmd[cmd.index("--output_path") + 1]) + assert len(out_dirs) == 2 # no output filename collision possible + + def test_metrics_cover_all_shards(self, rr_env, tmp_path): + output_path = self._run(rr_env, tmp_path) + + result = json.loads(output_path.read_text()) + metrics = result["results"][RD_TASK_REFCOCOG] + # gIoU over all four turns from both shards: (1.0+0.0+0.5+0.8)/4 + assert metrics["gIoU"] == pytest.approx(0.575) + # per-round: R1 = mean(1.0, 0.5), R2 = mean(0.0, 0.8) + assert metrics["gIoU_R1"] == pytest.approx(0.75) + assert metrics["gIoU_R2"] == pytest.approx(0.4) + + def test_empty_shard_output_is_refused(self, rr_env, tmp_path): + """Regression for the --idx/--num_parts bug: a shard that silently + evaluates nothing must fail the run, not shrink the benchmark.""" + _FakeInferencePopen.write_empty_for_dir = {"shard_1"} + with pytest.raises(RuntimeError, match="subset|partial"): + self._run(rr_env, tmp_path) + + def test_failed_shard_reported_after_all_finish(self, rr_env, tmp_path): + _FakeInferencePopen.returncode_for_dir = {"shard_1": 3} + with pytest.raises(RuntimeError, match="shard 1 exited with code 3"): + self._run(rr_env, tmp_path) diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index 083f12bd..bd6802d5 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -57,7 +57,7 @@ def test_schedule_evals_slurm_template_var_overrides(tmp_path): skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, - slurm_template_var='{"PARTITION":"dev-g","ACCOUNT":"myproject","TIME":"02:15:00","GPUS_PER_NODE":2}', + slurm_template_var='{"PARTITION":"dev-g","ACCOUNT":"myproject","TIME":"02:15:00","GPUS_PER_NODE":2,"SLURM_MEM":"123G"}', ) sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) @@ -67,6 +67,9 @@ def test_schedule_evals_slurm_template_var_overrides(tmp_path): assert "#SBATCH --account=myproject" in sbatch_content assert "#SBATCH --time=02:15:00" in sbatch_content assert "#SBATCH --gres=gpu:2" in sbatch_content + # SLURM_MEM is not modeled by SlurmOverrides — it must survive the + # EvalConfig round-trip via the extra_template_vars passthrough. + assert "#SBATCH --mem=123G" in sbatch_content def test_schedule_evals_slurm_template_var_invalid_json(tmp_path): @@ -96,3 +99,114 @@ def test_schedule_evals_slurm_template_var_invalid_json(tmp_path): dry_run=True, slurm_template_var='["partition", "dev-g"]', ) + + +def test_schedule_evals_dry_run_with_full_queue(tmp_path): + """A full queue must not crash a --dry-run (ZeroDivisionError regression).""" + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=5), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path), "QUEUE_LIMIT": "5"}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + sbatch_files = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch_files) == 1 + assert "#SBATCH --array=0-0%" in sbatch_files[0].read_text() + + +def test_schedule_evals_rejects_comma_in_model_path(tmp_path): + """Comma-bearing fields would be shredded by the bash CSV reader — refuse early.""" + csv_path = tmp_path / "jobs.csv" + csv_path.write_text( + "model_path,task_path,n_shot,eval_suite\n" + '"EleutherAI/pythia-160m,revision=step100000",hellaswag,0,lm_eval\n' + ) + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + with pytest.raises(ValueError, match="comma"): + schedule_evals( + eval_csv_path=str(csv_path), + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + +def test_schedule_evals_sbatch_failure_exits_nonzero(tmp_path): + """sbatch submission failure must surface as a non-zero exit, not exit 0.""" + import subprocess as _subprocess + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + patch( + "oellm.scheduler.subprocess.run", + side_effect=FileNotFoundError("sbatch"), + ), + ): + with pytest.raises(SystemExit) as excinfo: + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=False, + ) + assert excinfo.value.code == 1 + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + patch( + "oellm.scheduler.subprocess.run", + side_effect=_subprocess.CalledProcessError(1, ["sbatch"]), + ), + ): + with pytest.raises(SystemExit) as excinfo: + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=False, + ) + assert excinfo.value.code == 1 + + +def test_schedule_evals_local_failure_exits_nonzero(tmp_path): + """A failed --local run must propagate the script's exit code.""" + import subprocess as _subprocess + + with ( + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + patch( + "oellm.scheduler.subprocess.run", + side_effect=_subprocess.CalledProcessError(7, ["bash"]), + ), + ): + with pytest.raises(SystemExit) as excinfo: + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="hellaswag", + n_shot=0, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + local=True, + dry_run=False, + ) + assert excinfo.value.code == 7 From f05c263b2cae799ab74bdb77027a5ab4bffe228b Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Thu, 18 Jun 2026 09:57:43 +0200 Subject: [PATCH 26/44] bug fixes --- oellm/results.py | 11 +++++++++++ oellm/utils.py | 40 +++++++++++++++++----------------------- tests/test_utils.py | 35 ++++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/oellm/results.py b/oellm/results.py index 45b667eb..b2956b27 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -332,6 +332,17 @@ def collect_results( ) results = data.get("results", {}) + if not isinstance(results, dict): + # Not a standard eval-result file — e.g. this tool's own + # eval_results.json output (whose `results` is a list of rows) if it + # was written into a scanned directory, or any other non-eval JSON + # the recursive *.json discovery picked up. Skip it so re-running + # collect in place doesn't ingest its own output. + logging.debug( + f"Skipping '{json_file}': 'results' is a " + f"{type(results).__name__}, not a dict" + ) + continue n_shot_data = data.get("n-shot", {}) # lmms-eval has no "n-shot" dict; fall back to per-task config "num_fewshot" diff --git a/oellm/utils.py b/oellm/utils.py index 5a656d7e..361adfd2 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -441,11 +441,16 @@ def _pre_download_datasets_from_specs( label = f"{spec.repo_id}" + (f"/{spec.subset}" if spec.subset else "") status.update(f"Downloading '{label}' ({idx}/{len(specs_list)})") - snapshot_failed = False + # Media datasets (audio/video/image → needs_snapshot_download) keep + # their media as SEPARATE files in the repo (video .mp4s, audio + # clips) that load_dataset() does NOT fetch — it only builds the + # parquet/QA. snapshot_download the full repo (every revision, e.g. + # OpenGVLab/MVBench's `video` branch) so those media files are + # present on the offline compute node; load_dataset() below then + # builds the Arrow cache from the same files. BOTH are required: + # snapshot for the media assets, load_dataset for offline-loadability. revisions = getattr(spec, "revisions", None) or ["main"] if spec.needs_snapshot_download: - # Iterate every requested revision (e.g. OpenGVLab/MVBench - # splits content across `main` and `video` branches). for rev in revisions: rev_label = f"{label}@{rev}" if rev != "main" else label status.update(f"Downloading '{rev_label}' ({idx}/{len(specs_list)})") @@ -459,24 +464,18 @@ def _pre_download_datasets_from_specs( max_workers=2, ) except Exception as e: - snapshot_failed = True logging.warning( - f"snapshot_download failed for '{rev_label}': {e}; " - f"falling back to load_dataset." + f"snapshot_download failed for '{rev_label}': {e}" ) - if spec.needs_snapshot_download and not snapshot_failed: - # Snapshot already cached the raw files. Skip the - # load_dataset() build below: it fully materializes the - # dataset into Arrow (decoding every image/audio sample), - # which OOM-kills large media datasets on the memory-capped - # login node. The compute node builds at runtime from the - # cached files (offline-safe and has far more RAM). - logging.debug( - f"Snapshot complete for '{label}'; skipping load_dataset build." - ) - continue - + # Build the Arrow cache — this is what makes the dataset loadable on the + # OFFLINE compute nodes. load_dataset() needs the BUILT dataset under + # HF_DATASETS_CACHE; a bare hub snapshot is not loadable offline (it + # still tries to reach the Hub for the dataset module → ConnectionError). + # load_dataset reuses any files already fetched above — it does not + # re-download them. NOTE: the build can OOM the login node for very + # large media datasets (e.g. 60 GB librispeech), which then need a + # separate staging strategy. try: ds = load_dataset( spec.repo_id, @@ -528,11 +527,6 @@ def _pre_download_datasets_from_specs( logging.debug(f"Finished downloading dataset '{label}'.") continue - if snapshot_failed: - logging.debug( - f"Both snapshot_download and load_dataset failed for '{label}'." - ) - if failures: details = "\n".join( f" - {label}: {type(e).__name__}: {e}" for label, e in failures diff --git a/tests/test_utils.py b/tests/test_utils.py index 28fda323..ca7374f1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -173,9 +173,10 @@ class TestPreDownloadRevisions: """Tests for DatasetSpec.revisions handling in pre-download. Datasets like OpenGVLab/MVBench split content across branches: parquet - metadata on `main`, video assets on `video`. snapshot_download must be - called once per revision; the default of ["main"] preserves the - legacy single-snapshot behavior for all other datasets. + metadata on `main`, video assets on `video`. For media datasets + (needs_snapshot_download) snapshot_download runs once per revision so every + branch's files — including separate video/audio assets that load_dataset() + does not fetch — are present on the offline compute node. """ def test_snapshot_download_called_once_per_revision(self, monkeypatch): @@ -201,13 +202,16 @@ def fake_load_dataset(*args, **kwargs): _pre_download_datasets_from_specs(specs) + # Media datasets snapshot EVERY revision so each branch's files (incl. + # the separate video/audio assets) are fetched; load_dataset() then + # builds the Arrow cache. assert snapshot_calls == [ ("OpenGVLab/MVBench", "main"), ("OpenGVLab/MVBench", "video"), ] def test_default_revisions_falls_back_to_main(self, monkeypatch): - """Specs without an explicit revisions list still snapshot 'main'.""" + """A media spec without an explicit revisions list still snapshots 'main'.""" snapshot_calls = [] def fake_snapshot(*, repo_id, repo_type, revision, max_workers): @@ -220,13 +224,34 @@ def fake_load_dataset(*args, **kwargs): monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot) monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) - # Spec with default revisions=["main"]. + # Media spec with default revisions=["main"]. specs = [_FakeSpec(repo_id="some-org/dataset", needs_snapshot_download=True)] _pre_download_datasets_from_specs(specs) assert snapshot_calls == [("some-org/dataset", "main")] + def test_non_media_spec_is_not_snapshotted(self, monkeypatch): + """Text datasets (needs_snapshot_download=False) skip snapshot entirely — + load_dataset() fetches + builds them in one step.""" + snapshot_calls = [] + + def fake_snapshot(*, repo_id, repo_type, revision, max_workers): + snapshot_calls.append((repo_id, revision)) + + def fake_load_dataset(*args, **kwargs): + return object() + + monkeypatch.setattr("oellm.utils.get_console", lambda: _NoopConsole()) + monkeypatch.setattr("huggingface_hub.snapshot_download", fake_snapshot) + monkeypatch.setattr("datasets.load_dataset", fake_load_dataset) + + specs = [_FakeSpec(repo_id="text-org/dataset", needs_snapshot_download=False)] + + _pre_download_datasets_from_specs(specs) + + assert snapshot_calls == [] + class TestMaterializeExternalUrls: """`_materialize_external_urls` forces HF datasets' lazy URL fetches by From 2bc1e544e5fbe32cd3cdb4eff0bbdd893a16bb2f Mon Sep 17 00:00:00 2001 From: islobozhan Date: Fri, 19 Jun 2026 13:37:06 +0200 Subject: [PATCH 27/44] Fix textvqa offline staging, collect robustness, and ufal cluster account Fix textvqa offline staging, collect robustness, and ufal cluster account --- oellm/resources/task-groups.yaml | 4 +- oellm/results.py | 10 +++++ oellm/task_groups.py | 5 ++- oellm/utils.py | 10 ++++- tests/test_collect_results.py | 28 +++++++++++++ tests/test_image_task_groups.py | 2 +- tests/test_utils.py | 72 +++++++++++++++++++++++++++++++- 7 files changed, 124 insertions(+), 7 deletions(-) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 4aa546b1..7d452d54 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -459,7 +459,7 @@ task_groups: dataset: lmms-lab/DocVQA subset: DocVQA - task: textvqa_val - dataset: facebook/textvqa + dataset: lmms-lab/textvqa - task: ocrbench dataset: echo840/OCRBench - task: mathvista_testmini_cot @@ -517,7 +517,7 @@ task_groups: n_shots: [0] tasks: - task: textvqa_val - dataset: facebook/textvqa + dataset: lmms-lab/textvqa image-ocrbench: description: "OCRBench optical character recognition benchmark via lmms-eval" diff --git a/oellm/results.py b/oellm/results.py index b2956b27..ec070b87 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -315,6 +315,16 @@ def collect_results( ) continue + # Some files the recursive *.json scan finds are not result objects at + # all — e.g. lmms-eval per-sample logs (a bare JSON array) or other + # non-eval JSON. Skip anything that isn't a dict before calling .get(). + if not isinstance(data, dict): + logging.debug( + f"Skipping '{json_file}': top-level JSON is a " + f"{type(data).__name__}, not a dict" + ) + continue + # Model name lives in different keys depending on the harness: # - lmms-eval: model_name_or_path is the checkpoint, model_name is the # adapter class (e.g. "llava_hf") diff --git a/oellm/task_groups.py b/oellm/task_groups.py index d0436c9c..0adbd6d0 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -11,7 +11,10 @@ # snapshot_download only fetches the URLs, not the media — they MUST go through # load_dataset()+_materialize_external_urls() so the per-row HTTP fetch runs on # the (online) login node. Excluded from the snapshot-only fast path below. -_URL_BASED_DATASETS = {"facebook/textvqa"} +# Currently empty: every in-tree dataset embeds its media (e.g. lmms-lab/textvqa +# ships image bytes in its parquet). Add a repo id here only if it genuinely +# stores external media URLs. +_URL_BASED_DATASETS: set[str] = set() # Eval suites whose datasets are large media (image/audio/video). Their specs # are staged with snapshot_download (raw files; the compute node builds the diff --git a/oellm/utils.py b/oellm/utils.py index 7f3672f1..5d9ef2eb 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -178,12 +178,18 @@ def __missing__(self, key): # would resolve caches to "/datasets" and fail far from the real cause. _required_vars = [ "PARTITION", - "ACCOUNT", "EVAL_BASE_DIR", "EVAL_OUTPUT_DIR", "GPUS_PER_NODE", "HF_HOME", ] + # ACCOUNT is required only for clusters that declare it. Some clusters + # (e.g. ufal) intentionally omit it and rely on the submitting user's + # default SLURM account; for those the sbatch's `#SBATCH --account=$ACCOUNT` + # directive is stripped entirely in schedule_evals(). If a cluster does + # declare ACCOUNT, it is still validated (including unresolved "{...}"). + if "ACCOUNT" in final_env: + _required_vars.append("ACCOUNT") missing = [ v for v in _required_vars if not os.environ.get(v) or "{" in os.environ.get(v, "") ] @@ -395,7 +401,7 @@ def _pre_download_hf_dataset_files(dataset_files: list[dict]) -> None: def _materialize_external_urls(ds, *, max_workers: int = 16) -> None: """Iterate every row to force HF ``dl_manager`` to fetch external URLs. - Datasets like ``facebook/textvqa`` store image URLs (not bytes) in + Some datasets store media as external URLs (not bytes) in parquet rows; only per-row access triggers the HTTP fetch into the cache. Strict: exceptions propagate so ``_pre_download_datasets_…`` aborts the schedule before SLURM submission. diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 2a19598b..0ed3c1fd 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -508,6 +508,34 @@ def test_truncated_json_is_skipped(self, tmp_path): assert len(df) == 1 assert df.iloc[0]["performance"] == pytest.approx(0.75) + def test_top_level_list_json_is_skipped(self, tmp_path): + # lmms-eval writes per-sample logs as a bare JSON array; the recursive + # *.json scan picks them up. They must be skipped, not crash collect + # with "'list' object has no attribute 'get'". + results_dir = tmp_path / "results" + results_dir.mkdir() + write_result( + results_dir, + { + "model_name": "/path/to/model", + "results": {"textvqa_val": {"exact_match,none": 0.6}}, + "n-shot": {"textvqa_val": 0}, + }, + filename="good.json", + ) + # A bare JSON array, as lmms-eval emits for per-sample logs. + (results_dir / "samples.json").write_text( + json.dumps([{"doc_id": 0, "pred": "a"}, {"doc_id": 1, "pred": "b"}]) + ) + + output_csv = str(tmp_path / "out.csv") + collect_results(str(results_dir), output_csv=output_csv) + + df = pd.read_csv(output_csv) + assert len(df) == 1 + assert df.iloc[0]["task"] == "textvqa_val" + assert df.iloc[0]["performance"] == pytest.approx(0.6) + def test_corrupt_json_counts_as_missing_in_check(self, tmp_path): (tmp_path / "jobs.csv").write_text( "model_path,task_path,n_shot,eval_suite\n/models/pythia-160m,mmlu,5,lm_eval\n" diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py index 127398bf..62c1f9e9 100644 --- a/tests/test_image_task_groups.py +++ b/tests/test_image_task_groups.py @@ -34,7 +34,7 @@ "MMMU/MMMU", "lmms-lab/ChartQA", "lmms-lab/DocVQA", - "facebook/textvqa", + "lmms-lab/textvqa", "echo840/OCRBench", "AI4Math/MathVista", } diff --git a/tests/test_utils.py b/tests/test_utils.py index ca7374f1..a6c1b2c0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,12 @@ +import os from dataclasses import dataclass, field +from unittest.mock import patch import pytest from oellm.utils import ( _expand_local_model_paths, + _load_cluster_env, _materialize_external_urls, _num_jobs_in_queue, _pre_download_datasets_from_specs, @@ -257,7 +260,7 @@ class TestMaterializeExternalUrls: """`_materialize_external_urls` forces HF datasets' lazy URL fetches by accessing EVERY row of EVERY split. - Datasets like `facebook/textvqa` store image URLs as row fields rather + Some datasets store image URLs as row fields rather than embedded image bytes; the actual HTTP fetch is deferred until each row is read. We must trigger it here on the login node so the cache is complete before SLURM submission. Touching only the first row leaves @@ -470,3 +473,70 @@ def __exit__(self, *args): def update(self, *args, **kwargs): pass + + +class TestLoadClusterEnv: + """_load_cluster_env required-var validation, incl. the no-ACCOUNT cluster case. + + Uses a SYNTHETIC cluster config (not the real, user-customizable + clusters.yaml) so assertions exercise behavior rather than a specific + deployment's account/partition strings. + """ + + @staticmethod + def _clusters(): + # "noacct" mirrors ufal: a SLURM cluster that declares no ACCOUNT. + # "withacct" mirrors a standard cluster that declares one. + return { + "shared": { + "EVAL_OUTPUT_DIR": "{EVAL_BASE_DIR}/{USER}", + "GPUS_PER_NODE": 1, + }, + "withacct": { + "hostname_pattern": "*.withacct.test", + "EVAL_BASE_DIR": "/data/evals", + "PARTITION": "gpu", + "ACCOUNT": "proj-123", + }, + "noacct": { + "hostname_pattern": "*.noacct.test", + "EVAL_BASE_DIR": "/data/evals", + "PARTITION": "gpu-a,gpu-b", + }, + } + + def _run(self, monkeypatch, hostname, env): + # Pin the hostname AND the cluster config so the test is independent of + # the real clusters.yaml and the ambient shell env (restored afterwards). + monkeypatch.setattr("oellm.utils.socket.gethostname", lambda: hostname) + monkeypatch.setattr("oellm.utils.socket.getfqdn", lambda: hostname) + monkeypatch.setattr( + "oellm.utils.yaml.safe_load", lambda *_a, **_k: self._clusters() + ) + with patch.dict(os.environ, env, clear=True): + _load_cluster_env() + return dict(os.environ) + + def test_no_account_cluster_does_not_raise(self, monkeypatch, tmp_path): + # A cluster that declares no ACCOUNT (ufal-style) must NOT raise; the + # sbatch --account directive is stripped later when ACCOUNT is unset. + result = self._run( + monkeypatch, "node.noacct.test", {"USER": "tester", "HF_HOME": str(tmp_path)} + ) + assert result["PARTITION"] == "gpu-a,gpu-b" + assert "ACCOUNT" not in result + + def test_account_cluster_still_sets_account(self, monkeypatch, tmp_path): + # A cluster that declares ACCOUNT must still resolve and keep it. + result = self._run( + monkeypatch, + "node.withacct.test", + {"USER": "tester", "HF_HOME": str(tmp_path)}, + ) + assert result["ACCOUNT"] == "proj-123" + + def test_other_required_vars_still_enforced(self, monkeypatch): + # The fix must be surgical: a genuinely missing required var (HF_HOME) + # still raises, even on a no-ACCOUNT cluster. + with pytest.raises(RuntimeError, match="HF_HOME"): + self._run(monkeypatch, "node.noacct.test", {"USER": "tester"}) From 41eabb8a2ab0609f641dbf1bd50dc9d841376f3e Mon Sep 17 00:00:00 2001 From: islobozhan Date: Thu, 25 Jun 2026 18:08:47 +0200 Subject: [PATCH 28/44] [Base][Image] Add image VL benchmarks Add image VL benchmarks: OCRBench v2, RealWorldQA, MMStar, AI2D, MathVision, MME-RealWorld, SEED-Bench --- README.md | 10 ++++- oellm/resources/task-groups.yaml | 65 ++++++++++++++++++++++++++++++++ oellm/results.py | 6 +++ pyproject.toml | 11 ++++++ 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94a440bc..7637ffda 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Diagnose your environment** (cluster vars, HF cache, venv engines): `oellm-eval doctor` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Jupiter, Snellius) -- **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, MathVista) +- **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, OCRBench v2, MathVista, MathVision, MMStar, AI2D, RealWorldQA, MME, MME-RealWorld, SEED-Bench) - **Video evaluation** via lmms-eval (VideoMMMU, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench) - **Audio evaluation** via lmms-eval (LibriSpeech, FLEURS, GigaSpeech, TED-LIUM, WenetSpeech, CoVoST2, VocalSound, MuChoMusic) - **Plugin system** for contributing custom benchmarks without touching core code @@ -87,7 +87,15 @@ Super groups: `oellm-multilingual` (all multilingual benchmarks combined) | `image-docvqa` | DocVQA | lmms-eval | | `image-textvqa` | TextVQA | lmms-eval | | `image-ocrbench` | OCRBench | lmms-eval | +| `image-ocrbench-v2` | OCRBench v2 | lmms-eval | | `image-mathvista` | MathVista (CoT / format / solution leaves — needs GPT judge) | lmms-eval | +| `image-mathvision` | MathVision (test split) | lmms-eval | +| `image-mme` | MME (perception + cognition) | lmms-eval | +| `image-mme-realworld` | MME-RealWorld (EN) | lmms-eval | +| `image-mmstar` | MMStar | lmms-eval | +| `image-ai2d` | AI2D (diagram QA) | lmms-eval | +| `image-realworldqa` | RealWorldQA | lmms-eval | +| `image-seedbench` | SEED-Bench (image split) | lmms-eval | ### Video diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 4f5bc2bd..84d1bac7 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -120,6 +120,14 @@ task_metrics: mathvista_testmini_cot: llm_as_judge_eval mathvista_testmini_format: llm_as_judge_eval mathvista_testmini_solution: llm_as_judge_eval + # Additional lmms-eval image benchmarks (rule-based, air-gap safe). + ocrbench_v2: ocrbench_v2_accuracy + realworldqa: exact_match + mmerealworld: mme_realworld_score + mmstar: average + ai2d: exact_match + mathvision_test: mathvision_standard_eval + seedbench: seed_image # lmms-eval video benchmark metrics. `video_mmmu` is a group of 3 leaves. video_mmmu_perception: mmmu_acc video_mmmu_comprehension: mmmu_acc @@ -545,6 +553,63 @@ task_groups: - task: mathvista_testmini_format - task: mathvista_testmini_solution + # ── Image Modality: additional benchmarks (lmms-eval, rule-based) ───────── + image-ocrbench-v2: + description: "OCRBench v2 OCR benchmark (rule-based) via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: ocrbench_v2 + dataset: ling99/OCRBench_v2 + + image-realworldqa: + description: "RealWorldQA real-world spatial/visual understanding via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: realworldqa + dataset: lmms-lab/RealWorldQA + + image-mme-realworld: + description: "MME-RealWorld (EN) high-resolution real-world MCQ via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mmerealworld + dataset: yifanzhang114/MME-RealWorld-Lmms-eval + + image-mmstar: + description: "MMStar elite vision-indispensable multimodal benchmark via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mmstar + dataset: Lin-Chen/MMStar + + image-ai2d: + description: "AI2D diagram multiple-choice VQA via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: ai2d + dataset: lmms-lab/ai2d + + image-mathvision: + description: "MathVision math reasoning over images (test split, rule-based) via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: mathvision_test + dataset: MathLLMs/MathVision + + image-seedbench: + description: "SEED-Bench image multiple-choice (seed_image submetric) via lmms-eval" + suite: lmms_eval + n_shots: [0] + tasks: + - task: seedbench + dataset: lmms-lab/SEED-Bench + # ── Video Modality (lmms-eval) ──────────────────────────────────────────── video-understanding: description: "Video understanding benchmarks via lmms-eval (VideoMMMU, MVBench, EgoSchema-subset, VideoMME, ActivityNet-QA, LongVideoBench)" diff --git a/oellm/results.py b/oellm/results.py index ec070b87..6b0883fb 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -37,6 +37,11 @@ "mer": 1.0, "f1": 1.0, "semantic_match": 1.0, + # added image benchmarks (0–1) + "ocrbench_v2_accuracy": 1.0, + "mme_realworld_score": 1.0, + "average": 1.0, # MMStar headline = mean of per-category accuracies + "seed_image": 1.0, # SEED-Bench image submetric # ── 0–100 scale (no scaling needed) ── "gpt_eval_score": 100.0, "llm_as_judge_eval": 100.0, @@ -46,6 +51,7 @@ "submission": 100.0, "bleu": 100.0, "chrf++": 100.0, + "mathvision_standard_eval": 100.0, # ── 0–5 Likert scale (GPT-judge style) ── "gpt_eval": 5.0, # ── Unbounded / non-standard ── diff --git a/pyproject.toml b/pyproject.toml index b2f1030f..b2e13ba3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,17 +38,28 @@ image = [ "transformers>=4.45,<4.50", # Phi-4-multimodal trust_remote_code modeling imports backoff. "backoff", + # lmms-eval's qwen2_vl / qwen2_5_vl adapters do a top-level `import decord` + # (it lives only in lmms-eval's optional `video-legacy` extra), so it must be + # present for ANY Qwen-VL run — image tasks included. Mirrors lmms-eval's own + # platform markers; no usable macOS wheel on Python >= 3.12. + "decord; platform_system != 'Darwin'", + "eva-decord; platform_system == 'Darwin' and python_version < '3.12'", ] video = [ "transformers>=4.45,<4.50", "backoff", # PyAV 14 renamed AVError to OSError; torchvision still uses the old name. "av<14", + # lmms-eval video adapters (qwen2_vl etc.) import decord at module load. + "decord; platform_system != 'Darwin'", + "eva-decord; platform_system == 'Darwin' and python_version < '3.12'", ] # HPC Singularity image must also include ffmpeg for non-WAV decode. audio = [ "soundfile", "librosa", + "numba>=0.61", + "llvmlite>=0.44", "jiwer", ] # AudioBench contrib plugin. AudioBench itself is not pip-installable From d3a13fc55b046fed750a3fba05e8b10b33fc1124 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Thu, 16 Jul 2026 11:59:25 +0200 Subject: [PATCH 29/44] [Base] Results correctness, quantization, timeouts * results correctness, quantization, timeouts * naming fixes --- containers/jupiter.def | 4 +- containers/jureca.def | 4 +- containers/leonardo.def | 4 +- containers/lumi.def | 4 +- containers/slurm-ci.def | 4 +- containers/snellius.def | 4 +- docs/VENV.md | 4 +- oellm/__init__.py | 3 + oellm/config.py | 20 ++ oellm/contrib/CONTRIBUTING.md | 12 + oellm/main.py | 53 ++- oellm/resources/task-groups.yaml | 100 ++++++ oellm/resources/template.sbatch | 44 ++- oellm/results.py | 245 ++++++++++---- oellm/runner.py | 15 +- oellm/scheduler.py | 148 ++++++++- oellm/task_groups.py | 57 ++++ oellm/utils.py | 50 ++- pyproject.toml | 6 +- scripts/pivot_results.py | 54 +++- tests/test_collect_results.py | 2 +- tests/test_collection_and_scheduling.py | 412 ++++++++++++++++++++++++ tests/test_metric_snapshots.py | 79 +++-- tests/test_quantization_and_timeout.py | 179 ++++++++++ tests/test_reporter.py | 4 +- tests/test_utils.py | 12 +- 26 files changed, 1365 insertions(+), 158 deletions(-) create mode 100644 tests/test_collection_and_scheduling.py create mode 100644 tests/test_quantization_and_timeout.py diff --git a/containers/jupiter.def b/containers/jupiter.def index 063bb9bd..ec806d8c 100644 --- a/containers/jupiter.def +++ b/containers/jupiter.def @@ -13,7 +13,7 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 export UV_TOOL_BIN_DIR=/usr/local/bin uv --version - uv pip install --system --break-system-packages lm-eval \ + uv pip install --system --break-system-packages lm-eval==0.4.12 \ wandb sentencepiece tiktoken accelerate # lighteval as isolated tool (avoids dependency conflicts) @@ -23,7 +23,7 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 uv tool install --python 3.12 \ --with "pillow" \ --with "torch<2.9" \ - "lighteval @ git+https://github.com/huggingface/lighteval.git" + "lighteval @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" uv pip install --system --break-system-packages "spacy>=3.0.0,<4.0.0" jieba "underthesea>=6.0.0" uv pip install --system --break-system-packages nltk diff --git a/containers/jureca.def b/containers/jureca.def index 5c50e48f..ff579501 100644 --- a/containers/jureca.def +++ b/containers/jureca.def @@ -13,7 +13,7 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ - lm-eval \ + lm-eval==0.4.12 \ transformers \ "datasets<4.0.0" \ wandb \ @@ -26,7 +26,7 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)" diff --git a/containers/leonardo.def b/containers/leonardo.def index 5ab55500..1e7f6fa6 100644 --- a/containers/leonardo.def +++ b/containers/leonardo.def @@ -13,7 +13,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ - lm-eval \ + lm-eval==0.4.12 \ transformers \ "datasets<4.0.0" \ wandb \ @@ -26,7 +26,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)" diff --git a/containers/lumi.def b/containers/lumi.def index d02c1a25..14d527ce 100644 --- a/containers/lumi.def +++ b/containers/lumi.def @@ -13,7 +13,7 @@ From: rocm/pytorch:rocm6.4.1_ubuntu24.04_py3.12_pytorch_release_2.7.1 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ - lm-eval \ + lm-eval==0.4.12 \ transformers \ "datasets<4.0.0" \ wandb \ @@ -26,7 +26,7 @@ From: rocm/pytorch:rocm6.4.1_ubuntu24.04_py3.12_pytorch_release_2.7.1 uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)" diff --git a/containers/slurm-ci.def b/containers/slurm-ci.def index c11350f1..1b3851bc 100644 --- a/containers/slurm-ci.def +++ b/containers/slurm-ci.def @@ -13,7 +13,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ - lm-eval \ + lm-eval==0.4.12 \ transformers \ "datasets<4.0.0" \ wandb \ @@ -26,7 +26,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)" diff --git a/containers/snellius.def b/containers/snellius.def index 69037e9d..3dd4e5e9 100644 --- a/containers/snellius.def +++ b/containers/snellius.def @@ -13,7 +13,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ - lm-eval \ + lm-eval==0.4.12 \ transformers \ "datasets<4.0.0" \ wandb \ @@ -26,7 +26,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time /opt/uv-tools/lighteval/bin/python -c "from lighteval.tasks.registry import Registry; Registry.load_all_task_configs(load_multilingual=True)" diff --git a/docs/VENV.md b/docs/VENV.md index c3dddd69..be258ec8 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -39,7 +39,7 @@ uv venv --python 3.12 /path/to/.venv # Pin a known-good commit (`...lmms-eval.git@#egg=...`) so two venvs # created on different days run the same engine — unpinned `main` drifts. uv pip install --python /path/to/.venv/bin/python \ - -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git#egg=lmms-eval" + -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git@45c766f60b6f8c153e4c72d06ca636e2db0ebcdb#egg=lmms-eval" # 3. Install oellm-cli with engine extras uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' @@ -48,7 +48,7 @@ uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ uv tool install --python 3.12 \ --with "langcodes[data]" --with "pillow" \ - "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" + "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" ``` Verify with: diff --git a/oellm/__init__.py b/oellm/__init__.py index e69de29b..063a16a5 100644 --- a/oellm/__init__.py +++ b/oellm/__init__.py @@ -0,0 +1,3 @@ +"""ELLIOT evaluation platform (import package ``oellm``, distribution ``oellm-eval``).""" + +__version__ = "0.2.0" diff --git a/oellm/config.py b/oellm/config.py index 959badfc..fdd68500 100644 --- a/oellm/config.py +++ b/oellm/config.py @@ -84,6 +84,13 @@ class EvalConfig: venv_path: str | None = None lm_eval_include_path: str | None = None local: bool = False + # Quantized model loading (bitsandbytes) for the HF-style engines + # (lm_eval / lmms_eval / evalchemy). NVIDIA-first: ROCm (LUMI) support + # depends on bitsandbytes' ROCm build. lighteval rows are unaffected and + # contrib suites only opt in via the OELLM_QUANTIZATION env var — the + # scheduler warns when such rows are scheduled with a flag set. + load_in_4bit: bool = False + load_in_8bit: bool = False # ---- SLURM overrides ---- slurm: SlurmOverrides = field(default_factory=SlurmOverrides) @@ -145,6 +152,8 @@ def from_cli_kwargs( venv_path: str | None = None, lm_eval_include_path: str | None = None, local: bool | None = None, + load_in_4bit: bool | None = None, + load_in_8bit: bool | None = None, slurm_template_var: str | None = None, ) -> EvalConfig: """Build an ``EvalConfig`` from the loose CLI parameters. @@ -193,6 +202,8 @@ def from_cli_kwargs( ("venv_path", venv_path), ("lm_eval_include_path", lm_eval_include_path), ("local", local), + ("load_in_4bit", load_in_4bit), + ("load_in_8bit", load_in_8bit), ): if value is not None: provided.add(field_name) @@ -245,6 +256,8 @@ def from_cli_kwargs( venv_path=venv_path, lm_eval_include_path=lm_eval_include_path, local=bool(local) if local is not None else False, + load_in_4bit=bool(load_in_4bit) if load_in_4bit is not None else False, + load_in_8bit=bool(load_in_8bit) if load_in_8bit is not None else False, slurm=slurm, ) cfg._cli_provided = provided @@ -266,6 +279,8 @@ def from_cli_kwargs( "venv_path", "lm_eval_include_path", "local", + "load_in_4bit", + "load_in_8bit", "slurm", } ) @@ -320,6 +335,8 @@ def _from_dict(cls, raw: dict[str, Any]) -> EvalConfig: venv_path=raw.get("venv_path"), lm_eval_include_path=raw.get("lm_eval_include_path"), local=bool(raw.get("local", False)), + load_in_4bit=bool(raw.get("load_in_4bit", False)), + load_in_8bit=bool(raw.get("load_in_8bit", False)), slurm=slurm, ) @@ -359,6 +376,9 @@ def merge(self, cli: EvalConfig) -> EvalConfig: def validate(self) -> None: """Raise ``ValueError`` on invalid or contradictory configuration.""" + if self.load_in_4bit and self.load_in_8bit: + raise ValueError("load_in_4bit and load_in_8bit are mutually exclusive.") + if self.eval_csv_path: if self.models or self.tasks or self.task_groups or self.n_shot: raise ValueError( diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index e968b123..77739375 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -278,3 +278,15 @@ See `tests/test_regiondial_bench.py` as a reference. - Point `HF_HOME` to a filesystem with sufficient space. Home directories on HPC clusters typically have small quotas (50 GB or less). + +## Runtime environment variables available to `run()` + +The dispatcher passes the full job environment via the ``env`` parameter. +Interim channels (a formal ``eval_args`` parameter is planned for a future +plugin-protocol revision): + +- ``LIMIT`` — sample cap requested via ``--limit`` (empty = no cap). +- ``OELLM_QUANTIZATION`` — ``"4bit"`` / ``"8bit"`` / empty. Plugins that can + honor quantized loading should read this; plugins that cannot should ignore + it (the scheduler already warns the operator that contrib rows run at full + precision). diff --git a/oellm/main.py b/oellm/main.py index 82d666cf..dacc1cea 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -46,6 +46,8 @@ def schedule_evals( venv_path: str | None = None, lm_eval_include_path: str | None = None, local: bool | None = None, + load_in_4bit: bool | None = None, + load_in_8bit: bool | None = None, slurm_template_var: str | None = None, allow_missing_judge: bool = False, nodelist: str | None = None, @@ -97,6 +99,13 @@ def schedule_evals( local: If True, run evaluations directly on the local machine using bash instead of submitting to SLURM. Requires --venv-path. Skips cluster environment detection and runs all evaluations sequentially in a single process. + load_in_4bit: Load models 4-bit quantized (bitsandbytes) — applies to the + lm_eval / lmms_eval / evalchemy engines via --model_args; lighteval rows + are unaffected and contrib suites only opt in via the OELLM_QUANTIZATION + env var (a warning lists any full-precision rows). Recorded in the run's + provenance.json. NVIDIA-first; ROCm (LUMI) support depends on + bitsandbytes' ROCm build. Mutually exclusive with load_in_8bit. + load_in_8bit: Same as load_in_4bit but 8-bit quantization. slurm_template_var: JSON object of template variable overrides. Use exact env var names (PARTITION, ACCOUNT, GPUS_PER_NODE, SLURM_MEM, NODES). "TIME" overrides the time limit. Example: '{"PARTITION":"dev-g","ACCOUNT":"FOO","TIME":"02:00:00","GPUS_PER_NODE":2,"SLURM_MEM":"96G","NODES":"1"}' @@ -126,6 +135,8 @@ def schedule_evals( venv_path=venv_path, lm_eval_include_path=lm_eval_include_path, local=local, + load_in_4bit=load_in_4bit, + load_in_8bit=load_in_8bit, slurm_template_var=slurm_template_var, ) @@ -162,6 +173,8 @@ def schedule_evals( slurm_template_var=cfg.slurm_template_var_json, allow_missing_judge=allow_missing_judge, nodelist=nodelist, + load_in_4bit=cfg.load_in_4bit, + load_in_8bit=cfg.load_in_8bit, ) @@ -244,7 +257,10 @@ def compare( def _load_results(path_str: str) -> list[dict]: p = Path(path_str) if p.is_dir(): - p = p / "results.json" + # collect_results writes eval_results.json; accept the legacy + # results.json name as a fallback. + candidate = p / "eval_results.json" + p = candidate if candidate.exists() else p / "results.json" if not p.exists(): raise FileNotFoundError(f"Results file not found: {p}") data = json.loads(p.read_text()) @@ -253,20 +269,33 @@ def _load_results(path_str: str) -> list[dict]: results_a = _load_results(result_a) results_b = _load_results(result_b) - # Index by (task, n_shot, metric) - def _index(results: list[dict]) -> dict[tuple, float]: + # Index by (model, task, n_shot, metric) — model identity matters: + # collected files routinely contain several models, and a model-less + # key would silently overwrite rows across models. + def _index(results: list[dict]) -> dict[tuple, float | None]: idx = {} for r in results: - key = (r.get("task", ""), r.get("n_shot", 0), r.get("metric", "")) - idx[key] = r.get("performance", 0.0) + key = ( + r.get("model", ""), + r.get("task", ""), + r.get("n_shot", 0), + r.get("metric", ""), + ) + idx[key] = r.get("performance") return idx idx_a = _index(results_a) idx_b = _index(results_b) - all_keys = sorted(set(idx_a.keys()) | set(idx_b.keys())) + # n_shot legitimately mixes ints and strings ("unknown") — sort on a + # stringified key to avoid TypeError. + all_keys = sorted( + set(idx_a.keys()) | set(idx_b.keys()), + key=lambda k: tuple(str(x) for x in k), + ) console = get_console() table = Table(title="Comparison") + table.add_column("Model", style="bold") table.add_column("Task", style="bold") table.add_column("N-shot", justify="right") table.add_column("Metric") @@ -274,9 +303,9 @@ def _index(results: list[dict]) -> dict[tuple, float]: table.add_column("B", justify="right") table.add_column("\u0394", justify="right") # Delta - for task, n_shot, metric in all_keys: - val_a = idx_a.get((task, n_shot, metric)) - val_b = idx_b.get((task, n_shot, metric)) + for model, task, n_shot, metric in all_keys: + val_a = idx_a.get((model, task, n_shot, metric)) + val_b = idx_b.get((model, task, n_shot, metric)) str_a = f"{val_a:.4f}" if val_a is not None else "\u2014" str_b = f"{val_b:.4f}" if val_b is not None else "\u2014" if val_a is not None and val_b is not None: @@ -284,7 +313,7 @@ def _index(results: list[dict]) -> dict[tuple, float]: str_delta = f"{delta:+.4f}" else: str_delta = "\u2014" - table.add_row(task, str(n_shot), metric, str_a, str_b, str_delta) + table.add_row(model, task, str(n_shot), metric, str_a, str_b, str_delta) console.print(table) @@ -357,6 +386,8 @@ def eval_command( venv_path: str | None = None, lm_eval_include_path: str | None = None, local: bool | None = None, + load_in_4bit: bool | None = None, + load_in_8bit: bool | None = None, slurm_template_var: str | None = None, allow_missing_judge: bool = False, nodelist: str | None = None, @@ -402,6 +433,8 @@ def eval_command( venv_path=venv_path, lm_eval_include_path=lm_eval_include_path, local=local, + load_in_4bit=load_in_4bit, + load_in_8bit=load_in_8bit, slurm_template_var=slurm_template_var, allow_missing_judge=allow_missing_judge, nodelist=nodelist, diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 17822d13..96a99086 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -428,6 +428,106 @@ task_metrics: polymath_pt_high: exact_match polymath_pt_top: exact_match + # Explicit headline-metric picks for groups that would otherwise ride the + # METRIC_FALLBACK_KEYS chain: + # * belebele (plain): acc — the paper convention; the _cf variants stay + # acc_norm. + # * mgsm: exact_match under the flexible-extract filter — CoT output + # formats vary across languages, and strict-match scores differ by + # several points. + # * gsm8k: exact_match under strict-match — the benchmark's original + # strict answer extraction; English output is format-stable, unlike + # multilingual mgsm above. + # * ifeval: prompt_level_strict_acc — the usual single-number pick. + # * The evalchemy-routed `reasoning` tasks (GPQADiamond, MATH500, + # LiveCodeBench, HumanEval, AIME24/25, AMC23) stay unmapped: output + # key shape unverified. The scheduler warns about unmapped tasks. + gsm8k: exact_match,strict-match + ifeval: prompt_level_strict_acc + mbpp: pass_at_1 + jeopardy: exact_match + # belebele-eu-5-shot (24 tasks) + belebele_bul_Cyrl: acc + belebele_ces_Latn: acc + belebele_dan_Latn: acc + belebele_deu_Latn: acc + belebele_ell_Grek: acc + belebele_eng_Latn: acc + belebele_est_Latn: acc + belebele_fin_Latn: acc + belebele_fra_Latn: acc + belebele_hrv_Latn: acc + belebele_hun_Latn: acc + belebele_ita_Latn: acc + belebele_lit_Latn: acc + belebele_lvs_Latn: acc + belebele_mlt_Latn: acc + belebele_nld_Latn: acc + belebele_nob_Latn: acc + belebele_pol_Latn: acc + belebele_por_Latn: acc + belebele_ron_Latn: acc + belebele_slk_Latn: acc + belebele_slv_Latn: acc + belebele_spa_Latn: acc + belebele_swe_Latn: acc + # flores-200-eu-to-eng (23 tasks) + # flores-200-eng-to-eu (23 tasks) + # global-mmlu-eu (18 tasks) + global_mmlu_full_cs: acc + global_mmlu_full_de: acc + global_mmlu_full_el: acc + global_mmlu_full_en: acc + global_mmlu_full_es: acc + global_mmlu_full_fr: acc + global_mmlu_full_he: acc + global_mmlu_full_it: acc + global_mmlu_full_lt: acc + global_mmlu_full_nl: acc + global_mmlu_full_pl: acc + global_mmlu_full_pt: acc + global_mmlu_full_ro: acc + global_mmlu_full_ru: acc + global_mmlu_full_sr: acc + global_mmlu_full_sv: acc + global_mmlu_full_tr: acc + global_mmlu_full_uk: acc + # include (25 tasks) + include_base_44_albanian: acc + include_base_44_armenian: acc + include_base_44_azerbaijani: acc + include_base_44_basque: acc + include_base_44_belarusian: acc + include_base_44_bulgarian: acc + include_base_44_croatian: acc + include_base_44_dutch: acc + include_base_44_estonian: acc + include_base_44_finnish: acc + include_base_44_french: acc + include_base_44_georgian: acc + include_base_44_german: acc + include_base_44_greek: acc + include_base_44_hungarian: acc + include_base_44_italian: acc + include_base_44_lithuanian: acc + include_base_44_north macedonian: acc + include_base_44_polish: acc + include_base_44_portuguese: acc + include_base_44_russian: acc + include_base_44_serbian: acc + include_base_44_spanish: acc + include_base_44_turkish: acc + include_base_44_ukrainian: acc + # generic-multilingual (3 tasks) + xcopa: acc + xstorycloze: acc + xwinograd: acc + # mgsm-eu (4 tasks) + mgsm_native_cot_de: exact_match,flexible-extract + mgsm_native_cot_en: exact_match,flexible-extract + mgsm_native_cot_es: exact_match,flexible-extract + mgsm_native_cot_fr: exact_match,flexible-extract + task_groups: sib200-eu: description: "SIB-200 European language topic classification tasks (0-shot)" diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index f5c1adb5..aff0f358 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -19,6 +19,9 @@ TOTAL_EVALS={total_evals} # via oellm.contrib.dispatch) can read it from os.environ. Built-in suites # below still interpolate $LIMIT directly into their CLI flags. export LIMIT="{limit}" +# Quantization request ("4bit" / "8bit" / empty). HF-style engines receive it +# via --model_args below; contrib plugins may opt in by reading this variable. +export OELLM_QUANTIZATION="{quantization}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" @@ -135,18 +138,31 @@ do suite_normalized=$(echo "${{eval_suite%%:*}}" | tr '[:upper:]' '[:lower:]') # Helper function to run Python commands in the appropriate environment + # Optional per-row wall-clock bound: set ROW_TIMEOUT (GNU timeout + # DURATION, e.g. "7200" or "2h") so a hung engine fails just its row + # (exit 124) instead of consuming the slice's whole TIME_LIMIT. + _maybe_timeout() {{ + if [ -n "${{ROW_TIMEOUT:-}}" ]; then + timeout --kill-after=60 "$ROW_TIMEOUT" "$@" + else + "$@" + fi + }} + + # stdin is redirected from /dev/null so no child can consume the CSV rows + # feeding the enclosing while-read loop. run_python() {{ if [ -n "$VENV_PATH" ]; then source "$VENV_PATH/bin/activate" - python "$@" + _maybe_timeout python "$@" < /dev/null else - singularity exec $SINGULARITY_ARGS \ + _maybe_timeout singularity exec $SINGULARITY_ARGS \ $SINGULARITY_ENV_ARGS \ $SINGULARITY_HOME_ARG \ --bind $BIND_PATHS \ $EVAL_SIF_PATH \ env CUDA_VISIBLE_DEVICES=$GPU_DEVICES \ - python "$@" + python "$@" < /dev/null fi }} @@ -156,7 +172,7 @@ do echo "----------------------------------------------------" echo "lm_eval Execution" run_python -m lm_eval --model hf \ - --model_args pretrained="$model_path",trust_remote_code=True \ + --model_args pretrained="$model_path",trust_remote_code=True{quantization_model_args} \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "{evals_dir}/$(openssl rand -hex 5).json" \ @@ -190,14 +206,14 @@ do if [ -n "$VENV_PATH" ]; then source "$VENV_PATH/bin/activate" - lighteval accelerate \ + _maybe_timeout lighteval accelerate \ "model_name=$model_path,trust_remote_code=True,{additional_model_args}" \ "$LIGHT_TASK_ARG" \ --load-tasks-multilingual \ --output-dir "$RESULTS_SUBDIR" \ - ${{LIMIT:+--max-samples $LIMIT}} + ${{LIMIT:+--max-samples $LIMIT}} < /dev/null else - singularity exec $SINGULARITY_ARGS \ + _maybe_timeout singularity exec $SINGULARITY_ARGS \ $SINGULARITY_ENV_ARGS \ $SINGULARITY_HOME_ARG \ --bind $BIND_PATHS \ @@ -208,7 +224,7 @@ do "$LIGHT_TASK_ARG" \ --load-tasks-multilingual \ --output-dir "$RESULTS_SUBDIR" \ - ${{LIMIT:+--max-samples $LIMIT}} + ${{LIMIT:+--max-samples $LIMIT}} < /dev/null fi rc=$? ;; @@ -234,7 +250,7 @@ do run_python -m lmms_eval \ --model "$_lmms_adapter" \ - --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args" \ + --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args{quantization_model_args}" \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ @@ -249,14 +265,14 @@ do ( source "$VENV_PATH/bin/activate" cd "$EVALCHEMY_WORK_DIR" || exit 1 - accelerate launch --num-processes "$GPUS_PER_NODE" --num-machines 1 \ + _maybe_timeout accelerate launch --num-processes "$GPUS_PER_NODE" --num-machines 1 \ $MULTI_GPU_FLAG -m eval.eval \ --model hf \ --tasks "$task_path" \ - --model_args "trust_remote_code=True,pretrained=$model_path" \ + --model_args "trust_remote_code=True,pretrained=$model_path{quantization_model_args}" \ --batch_size auto \ --output_path "$RESULTS_SUBDIR" \ - ${{LIMIT:+--limit $LIMIT}} + ${{LIMIT:+--limit $LIMIT}} < /dev/null ) rc=$? else @@ -280,7 +296,9 @@ do echo "----------------------------------------------------" if [ "$rc" -ne 0 ]; then FAILED_ROWS=$((FAILED_ROWS + 1)) - echo "[ERROR] Evaluation FAILED (exit=$rc) for model=$model_path task=$task_path n_shot=$n_shot suite=$eval_suite" + _timeout_note="" + if [ "$rc" -eq 124 ]; then _timeout_note=" (ROW_TIMEOUT=${{ROW_TIMEOUT:-}} exceeded)"; fi + echo "[ERROR] Evaluation FAILED (exit=$rc)$_timeout_note for model=$model_path task=$task_path n_shot=$n_shot suite=$eval_suite" else echo "Evaluation finished for model: $model_path" fi diff --git a/oellm/results.py b/oellm/results.py index da780bf4..53f5ce84 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -4,6 +4,7 @@ import json import logging +import subprocess from datetime import UTC, datetime from importlib.resources import files from pathlib import Path @@ -61,19 +62,43 @@ # lmms_eval/tasks/mme/utils.py::mme_aggregate_results. "mme_cognition_score": 800.0, "mme_perception_score": 2000.0, + # ── Contrib / additional 0–1 metrics ── + "gIoU": 1.0, + "meteor": 1.0, + "string_match": 1.0, +} + +# Per-task overrides for metric keys whose scale differs from the name-level +# default above. METRIC_NATIVE_SCALE is keyed by metric name only, but the +# same key is emitted on different scales by different benchmarks: +# lm-eval's squadv2 wraps the official squad_v2 metric (0–100 percentages) +# while coqa's token-F1 is 0–1, and lmms-eval's voicebench judge returns a +# raw 1–5 average while mathvista's llm_as_judge_eval is 0–100. Keyed by +# (task_name, metric_name_without_filter). +TASK_METRIC_SCALE_OVERRIDES: dict[tuple[str, str], float] = { + ("squadv2", "f1"): 100.0, + ("voicebench_commoneval", "llm_as_judge_eval"): 5.0, } -def _normalize_to_100(value: float | None, metric_name: str | None) -> float | None: +def _normalize_to_100( + value: float | None, metric_name: str | None, task_name: str | None = None +) -> float | None: """Normalize a metric value to a 0–100 scale for cross-benchmark display. - Returns ``None`` when the metric's native scale is unknown — caller - should fall back to the raw value rather than guess. + Per-task scale overrides (``TASK_METRIC_SCALE_OVERRIDES``) take precedence + over the name-level ``METRIC_NATIVE_SCALE`` default. Returns ``None`` when + the metric's native scale is unknown — caller should fall back to the raw + value rather than guess. """ if value is None or metric_name is None: return None clean = metric_name.split(",")[0] - scale = METRIC_NATIVE_SCALE.get(clean) + scale = None + if task_name is not None: + scale = TASK_METRIC_SCALE_OVERRIDES.get((task_name, clean)) + if scale is None: + scale = METRIC_NATIVE_SCALE.get(clean) if scale is None: return None return value * (100.0 / scale) @@ -272,6 +297,26 @@ def collect_results( else results_path ) json_files = [p for p in search_root.rglob("*.json") if p.is_file()] + # Sort oldest-first by mtime (path as tiebreak) so the last-wins dedup + # below deterministically keeps the NEWEST result when the same + # (model, task, n_shot, metric) key appears in multiple files. Re-runs + # write fresh random-hex filenames, so without this the survivor depended + # on filesystem enumeration order. + json_files.sort(key=lambda p: (p.stat().st_mtime, str(p))) + + # Run-provenance sidecars written by the scheduler (schedule-time config, + # resolved model revisions, template knobs). Embedded verbatim in the + # results JSON envelope so a collected number can be traced to its run. + run_provenance: list[dict] = [] + for _prov in sorted(results_path.rglob("provenance.json")): + try: + _pdata = json.loads(_prov.read_text()) + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: + logging.warning(f"Unreadable provenance sidecar {_prov}: {e}") + continue + if isinstance(_pdata, dict): + _pdata["_path"] = str(_prov) + run_provenance.append(_pdata) if not json_files: logging.warning(f"No JSON files found in {results_dir}") @@ -311,6 +356,9 @@ def collect_results( completed_jobs = set() for json_file in json_files: + # Provenance sidecars are consumed separately above, not result files. + if json_file.name == "provenance.json": + continue # A truncated file (OOM-killed / timed-out SLURM task) must not abort # the whole collection; skip it and let --check report the job missing. try: @@ -372,7 +420,12 @@ def collect_results( global_n_shot = _infer_global_n_shot(n_shot_data) - # Aggregate groups (lm-eval harness) + # Aggregate groups (lm-eval harness). Collect every TOP-LEVEL group — + # one that is not itself a subtask of another group — since a file can + # legitimately contain several independent groups plus standalone + # tasks. When a top-level group has no numeric metric (benchmark-style + # non-aggregating parents, e.g. lm-eval's `leaderboard`), descend into + # its child groups so the file still contributes rows. groups_map = data.get("groups", {}) group_subtasks_map = data.get("group_subtasks", {}) group_aggregate_names = set(groups_map.keys()) | set(group_subtasks_map.keys()) @@ -381,61 +434,88 @@ def collect_results( for _s in _subs: group_subtask_names.add(_s) - # Prefer only the first aggregate metric from groups (simplified) + rows_before_file = len(rows) + if groups_map: - group_name, group_results = next(iter(groups_map.items())) - orig_group_name = group_name - n_shot = n_shot_data.get(orig_group_name, "unknown") - if n_shot == "unknown": - for subtask_name in group_subtasks_map.get(orig_group_name, []): - if subtask_name in n_shot_data: - n_shot = n_shot_data[subtask_name] - break - if n_shot == "unknown" and global_n_shot is not None: - n_shot = global_n_shot - # Strip ``'|N'`` n-shot suffix from the group name, falling back - # to the parsed N when n_shot is still unknown. - group_name, parsed_n = _split_task_and_nshot(orig_group_name) - if n_shot == "unknown" and parsed_n is not None: - n_shot = parsed_n - performance, metric_name = _resolve_metric( - group_name, group_results, task_metrics - ) - if performance is not None: - if check: - completed_jobs.add((model_name, group_name, n_shot)) - rows.append( - { - "model_name": model_name, - "task": group_name, - "n_shot": n_shot, - "performance": performance, - "performance_normalized": _normalize_to_100( - performance, metric_name - ), - "metric_name": metric_name if metric_name is not None else "", - } + _queue = [g for g in groups_map if g not in group_subtask_names] + if not _queue: + # Defensive: odd/cyclic group metadata — consider every group. + _queue = list(groups_map) + _visited: set[str] = set() + while _queue: + orig_group_name = _queue.pop(0) + if orig_group_name in _visited: + continue + _visited.add(orig_group_name) + group_results = groups_map.get(orig_group_name, {}) + n_shot = n_shot_data.get(orig_group_name, "unknown") + if n_shot == "unknown": + for subtask_name in group_subtasks_map.get(orig_group_name, []): + if subtask_name in n_shot_data: + n_shot = n_shot_data[subtask_name] + break + if n_shot == "unknown" and global_n_shot is not None: + n_shot = global_n_shot + # Strip ``'|N'`` n-shot suffix from the group name, falling + # back to the parsed N when n_shot is still unknown. + group_name, parsed_n = _split_task_and_nshot(orig_group_name) + if n_shot == "unknown" and parsed_n is not None: + n_shot = parsed_n + performance, metric_name = _resolve_metric( + group_name, group_results, task_metrics ) - # Skip per-task iteration when groups are present - continue + if performance is not None: + if check: + completed_jobs.add((model_name, group_name, n_shot)) + rows.append( + { + "model_name": model_name, + "task": group_name, + "n_shot": n_shot, + "performance": performance, + "performance_normalized": _normalize_to_100( + performance, metric_name, group_name + ), + "metric_name": metric_name if metric_name is not None else "", + } + ) + else: + # Metric-less aggregate: descend into child groups so + # aggregating children are still collected. + for _child in group_subtasks_map.get(orig_group_name, []): + if _child in groups_map: + _queue.append(_child) for task_name, task_results in results.items(): # Skip the lighteval 'all' aggregate pseudo-task if task_name == "all": continue - # Skip entries already added from groups + # Skip entries already handled by the groups pass above if groups_map and task_name in group_aggregate_names: continue # Skip any lm-eval group subtasks; keep only aggregates if task_name in group_subtask_names: continue - # Skip MMLU subtasks - only keep the aggregate score - if task_name.startswith("mmlu_") and task_name != "mmlu": + # Skip MMLU subtasks — but only when this file actually contains + # the MMLU aggregate. A lone `mmlu_pro`-style task (a different + # benchmark that merely shares the prefix) must NOT be dropped. + if ( + task_name.startswith("mmlu_") + and task_name != "mmlu" + and ("mmlu" in results or "mmlu" in groups_map) + ): continue - # Skip Global MMLU subtasks - keep only aggregates like global_mmlu_full_pt - if task_name.startswith("global_mmlu_") and task_name.count("_") >= 4: + # Skip Global MMLU subtasks — a task is a subtask iff a shorter + # global_mmlu_* aggregate in this file prefixes it. Language codes + # may themselves contain underscores (global_mmlu_full_zh_hans), + # so counting underscores cannot distinguish the two. + if task_name.startswith("global_mmlu_") and any( + _other != task_name and task_name.startswith(_other + "_") + for _other in results + if isinstance(_other, str) and _other.startswith("global_mmlu_") + ): continue # Strip ``'|N'`` n-shot suffix from the task name; use parsed N @@ -471,6 +551,12 @@ def collect_results( child_metric_name = sub_metric if not child_values: continue + if len(child_values) < len(subtasks): + logging.warning( + f"Aggregate '{task_name_clean}' in {json_file.name}: only " + f"{len(child_values)}/{len(subtasks)} subtasks present — " + f"missing children are excluded from the mean" + ) performance = sum(child_values) / len(child_values) metric_name = child_metric_name else: @@ -489,7 +575,7 @@ def collect_results( "n_shot": n_shot, "performance": performance, "performance_normalized": _normalize_to_100( - performance, metric_name + performance, metric_name, task_name_clean ), "metric_name": metric_name if metric_name is not None else "", } @@ -503,6 +589,13 @@ def collect_results( f"— value may be null (LLM judge not configured?) or metric key missing from task_metrics" ) + if len(rows) == rows_before_file: + logging.warning( + f"Result file '{json_file}' contributed zero rows — every " + f"task/aggregate was skipped or lacked a numeric metric. " + f"--check will keep reporting its jobs as missing." + ) + if not rows and not check: logging.warning("No results extracted from JSON files") return @@ -515,14 +608,22 @@ def collect_results( # them to NaN and break the JSON envelope). _deduped: dict[tuple, dict] = {} for _row in rows: - _deduped[ - ( - _row.get("model_name"), - _row.get("task"), - _row.get("n_shot"), - _row.get("metric_name"), + _key = ( + _row.get("model_name"), + _row.get("task"), + _row.get("n_shot"), + _row.get("metric_name"), + ) + if _key in _deduped and _deduped[_key].get("performance") != _row.get( + "performance" + ): + logging.warning( + f"Duplicate results for model={_key[0]!r} task={_key[1]!r} " + f"n_shot={_key[2]!r} metric={_key[3]!r}: " + f"{_deduped[_key].get('performance')} superseded by " + f"{_row.get('performance')} (newest result file wins)" ) - ] = _row + _deduped[_key] = _row rows = list(_deduped.values()) df = pd.DataFrame(rows) @@ -533,7 +634,7 @@ def collect_results( output_stem = Path(output_csv).with_suffix("") json_path = Path(f"{output_stem}.json") md_path = Path(f"{output_stem}.md") - write_results_json(rows, json_path) + write_results_json(rows, json_path, run_provenance=run_provenance) write_results_markdown(rows, md_path) logging.info(f"Results JSON: {json_path}") logging.info(f"Results Markdown: {md_path}") @@ -609,17 +710,41 @@ def collect_results( # Structured output: versioned JSON and Markdown report # --------------------------------------------------------------------------- -SCHEMA_VERSION = "1.1" +SCHEMA_VERSION = "1.2" + + +def _collector_git_commit() -> str | None: + """Best-effort git commit of the oellm checkout running collect. + + Returns ``None`` when installed as a wheel or git is unavailable. + """ + try: + out = subprocess.run( + ["git", "-C", str(Path(__file__).resolve().parent), "rev-parse", "HEAD"], + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + return out.stdout.strip() or None def write_results_json( rows: list[dict], output_path: str | Path, + run_provenance: list[dict] | None = None, ) -> None: - """Write versioned JSON: {version, generated_at, results: [...]}. + """Write versioned JSON: {version, generated_at, provenance, results}. Each result row has `performance` (raw engine value) and - `performance_normalized` (0-100 via METRIC_NATIVE_SCALE, or null). + `performance_normalized` (0-100 via METRIC_NATIVE_SCALE plus per-task + overrides, or null). Schema v1.2 adds `oellm_version`, + `collector_git_commit`, per-run provenance under `runs`, and a reserved + extensible `metadata` namespace (future additive fields — e.g. safety / + compliance metadata — land there without a schema migration). """ output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) @@ -637,9 +762,15 @@ def write_results_json( } ) + from oellm import __version__ + envelope = { "version": SCHEMA_VERSION, "generated_at": datetime.now(UTC).isoformat(), + "oellm_version": __version__, + "collector_git_commit": _collector_git_commit(), + "metadata": {}, + "runs": run_provenance or [], "results": results, } diff --git a/oellm/runner.py b/oellm/runner.py index f0d050e6..151cfdfa 100644 --- a/oellm/runner.py +++ b/oellm/runner.py @@ -64,10 +64,21 @@ def resolve_suite(self, job: EvaluationJob) -> str: appended as ``lmms_eval:``. For contrib suites the registry's ``detect_model_flags()`` provides the same service. """ - suite = job.eval_suite - canonical = _ALIAS_MAP.get(suite, suite) + raw = str(job.eval_suite) + suite = raw.strip() + # Normalise the engine head for the alias lookup: CSV-sourced suites + # may arrive mixed-case or whitespace-padded (``LMMS_EVAL``, + # `` lmms_eval``). envcheck normalises before validating, so without + # this the row would pass pre-flight and crash on the compute node + # after queue wait and model load. + head, _sep, tail = suite.partition(":") + canonical = _ALIAS_MAP.get(head.strip().lower(), suite) if canonical == "lmms_eval": + if tail: + # Already resolved (e.g. a --check re-schedule CSV round + # trip) — keep the adapter, normalise the head. + return f"lmms_eval:{tail}" adapter = detect_lmms_model_type(str(job.model_path)) resolved = f"lmms_eval:{adapter}" logging.debug("lmms-eval adapter for %s: %s", job.model_path, adapter) diff --git a/oellm/scheduler.py b/oellm/scheduler.py index da5e5816..4160af2d 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -3,6 +3,7 @@ import math import os import re +import socket import subprocess from datetime import datetime from importlib.resources import files @@ -11,7 +12,9 @@ import pandas as pd +from oellm import __version__ from oellm.constants import EvaluationJob +from oellm.results import _collector_git_commit, _load_task_metrics from oellm.runner import EvalRunner from oellm.task_groups import ( _build_task_suite_map, @@ -20,6 +23,8 @@ _collect_hf_model_repos, _expand_task_groups, _lookup_dataset_specs_for_tasks, + _lookup_hf_dataset_files_for_tasks, + _lookup_hf_model_repos_for_tasks, split_group_tokens, ) from oellm.utils import ( @@ -99,6 +104,28 @@ def _resolve_additional_model_args(local: bool = False) -> str: return f"batch_size={batch_size_value}" +def _probe_engine_versions(venv_path: str | None) -> dict[str, str]: + """Best-effort versions of the score-relevant packages in the venv that + will run the jobs, recorded into provenance.json. Venvs are per-user, + per-cluster state that nothing else records — this is what makes a + collected number traceable to the engine that produced it. Container + mode returns {} (versions live in the image; its name is recorded + separately).""" + if not venv_path: + return {} + python_bin = Path(venv_path).expanduser() / "bin" / "python" + if not python_bin.exists(): + return {} + from oellm.envcheck import probe_import + + versions: dict[str, str] = {} + for module in ("lm_eval", "lighteval", "lmms_eval", "transformers", "torch"): + ok, ver = probe_import(python_bin, module) + if ok: + versions[module] = ver + return versions + + @capture_third_party_output_from_kwarg("verbose") def schedule_evals( models: str | None = None, @@ -120,6 +147,8 @@ def schedule_evals( slurm_template_var: str | None = None, allow_missing_judge: bool = False, nodelist: str | None = None, + load_in_4bit: bool = False, + load_in_8bit: bool = False, ) -> None: """ Schedule evaluation jobs for a given set of models, tasks, and number of shots. @@ -163,6 +192,9 @@ def schedule_evals( """ _setup_logging(verbose) + if load_in_4bit and load_in_8bit: + raise ValueError("load_in_4bit and load_in_8bit are mutually exclusive.") + if local: if not venv_path: raise ValueError( @@ -302,6 +334,52 @@ def schedule_evals( runner = EvalRunner() runner.prepare_jobs(expanded_eval_jobs) + # Warn when a scheduled task has no explicit task_metrics entry — + # collect_results then relies on METRIC_FALLBACK_KEYS, whose choice is + # insertion-order-dependent for multi-filter engine outputs. + _tm = _load_task_metrics() + _unmapped_tasks = sorted( + {str(j.task_path) for j in expanded_eval_jobs if str(j.task_path) not in _tm} + ) + if _unmapped_tasks: + _shown = ", ".join(_unmapped_tasks[:10]) + if len(_unmapped_tasks) > 10: + _shown += f" (+{len(_unmapped_tasks) - 10} more)" + logging.warning( + f"No task_metrics entry for: {_shown} — collect will use fallback " + f"metric keys; add entries to task-groups.yaml for a stable metric policy." + ) + + # Quantized loading: applied via --model_args for the HF-style + # engines only. Never silent for the rest — a table mixing 4-bit and + # full-precision rows is a comparability trap, so unsupported suites are + # announced here and the choice is recorded in provenance.json. + quantization = "4bit" if load_in_4bit else "8bit" if load_in_8bit else "" + quantization_model_args = f",load_in_{quantization}=True" if quantization else "" + if quantization: + _quant_ok = { + "lm_eval", + "lm-eval", + "lm-eval-harness", + "lmms_eval", + "lmms-eval", + "evalchemy", + } + _unsupported = sorted( + { + str(j.eval_suite).split(":", 1)[0].strip().lower() + for j in expanded_eval_jobs + } + - _quant_ok + ) + if _unsupported: + logging.warning( + f"load_in_{quantization} applies to lm_eval/lmms_eval/evalchemy " + f"only; rows for suite(s) {', '.join(_unsupported)} run at FULL " + f"precision (contrib suites may opt in via the " + f"OELLM_QUANTIZATION env var exported to the job)." + ) + if not skip_checks: # Verify the runtime can actually execute the scheduled suites before # any network work: missing engines otherwise fail row-by-row on the @@ -320,8 +398,11 @@ def schedule_evals( for job in expanded_eval_jobs if not Path(job.model_path).exists() } - _process_model_paths(hub_models) + _revs = _process_model_paths(hub_models) + # Defensive: tests/legacy callers may stub this with a non-dict. + model_revisions = _revs if isinstance(_revs, dict) else {} else: + model_revisions = {} logging.info( "Skipping model path processing and validation (--skip-checks enabled)" ) @@ -375,15 +456,19 @@ def _lower_suite_only(s: str) -> str: dataset_specs, trust_remote_code=trust_remote_code ) - hf_model_repos = [] + # Auxiliary model repos / dataset files declared by tasks must be + # staged regardless of how the tasks were scheduled — task groups, + # bare --tasks, or a --check re-schedule CSV — because compute + # nodes run air-gapped (HF_HUB_OFFLINE=1). if group_names: hf_model_repos = _collect_hf_model_repos(group_names) + hf_dataset_files = _collect_hf_dataset_files(group_names) + else: + _all_task_names = df["task_path"].unique().tolist() + hf_model_repos = _lookup_hf_model_repos_for_tasks(_all_task_names) + hf_dataset_files = _lookup_hf_dataset_files_for_tasks(_all_task_names) if hf_model_repos: _pre_download_hf_model_repos(hf_model_repos) - - hf_dataset_files = [] - if group_names: - hf_dataset_files = _collect_hf_dataset_files(group_names) if hf_dataset_files: _pre_download_hf_dataset_files(hf_dataset_files) else: @@ -435,6 +520,17 @@ def _lower_suite_only(s: str) -> str: time_limit = os.environ.get("TIME_LIMIT", "12:00:00") + # On low-QUEUE_LIMIT clusters many rows run serially inside one + # wall-clock budget; without a per-row bound a single hung engine consumes + # the rest of the slice invisibly. + if evals_per_job > 1 and not os.environ.get("ROW_TIMEOUT"): + logging.warning( + f"{evals_per_job} evaluations run serially per array task under one " + f"TIME_LIMIT={time_limit} with no per-row timeout — one hung row " + f"consumes the rest of its slice. Set ROW_TIMEOUT (GNU timeout " + f"duration, e.g. '7200' or '2h') to bound each row." + ) + # Apply slurm_template_var overrides (JSON object) if slurm_template_var: try: @@ -472,6 +568,35 @@ def _lower_suite_only(s: str) -> str: logging.info(f" Time limit: {time_limit}") logging.info(f" Requested host memory: {slurm_mem}") + # Run-provenance sidecar: schedule-time config, resolved model revisions, + # and every knob that changes effective eval conditions. Picked up by + # collect_results and embedded in the results JSON envelope (v1.2) so a + # collected number can be traced back to its run. + provenance = { + "schema": 1, + "created_at": timestamp, + "hostname": socket.gethostname(), + "oellm_version": __version__, + "scheduler_git_commit": _collector_git_commit(), + "eval_suites": sorted({str(j.eval_suite) for j in expanded_eval_jobs}), + "total_evals": total_evals, + "array_size": actual_array_size, + "time_limit": time_limit, + "slurm_mem": slurm_mem, + "lighteval_model_args": additional_model_args, + "max_num_frames": os.environ.get("MAX_NUM_FRAMES"), + "limit": limit, + "venv_path": venv_path, + "hf_hub_offline": _resolve_hf_hub_offline(local), + "quantization": quantization or None, + "row_timeout": os.environ.get("ROW_TIMEOUT"), + "model_revisions": model_revisions, + "engine_versions": {} if skip_checks else _probe_engine_versions(venv_path), + "container_image": None if venv_path else os.environ.get("EVAL_CONTAINER_IMAGE"), + } + (evals_dir / "provenance.json").write_text(json.dumps(provenance, indent=2)) + logging.info(f"Run provenance: {evals_dir / 'provenance.json'}") + sbatch_script = sbatch_template.format( csv_path=csv_path, max_array_len=max_array_len, @@ -489,6 +614,8 @@ def _lower_suite_only(s: str) -> str: hf_hub_offline=_resolve_hf_hub_offline(local), additional_model_args=additional_model_args, evalchemy_dir=os.environ.get("EVALCHEMY_DIR", "/opt/evalchemy"), + quantization=quantization, + quantization_model_args=quantization_model_args, ) # Drop optional #SBATCH directives whose env var is unset, so safe_substitute @@ -503,8 +630,13 @@ def _lower_suite_only(s: str) -> str: if not os.environ.get("NODELIST"): sbatch_script = sbatch_script.replace("#SBATCH --nodelist=$NODELIST\n", "") - # substitute any $ENV_VAR occurrences - sbatch_script = Template(sbatch_script).safe_substitute(os.environ) + # Substitute $ENV_VAR occurrences from the environment — EXCLUDING SLURM_* + # runtime variables: when scheduling from inside an allocation + # (salloc/srun) those are set at render time and would be baked into the + # script (e.g. JOB_HOME losing its per-job uniqueness) instead of + # expanding on the compute node. + _template_env = {k: v for k, v in os.environ.items() if not k.startswith("SLURM_")} + sbatch_script = Template(sbatch_script).safe_substitute(_template_env) sbatch_script_path = evals_dir / "submit_evals.sbatch" diff --git a/oellm/task_groups.py b/oellm/task_groups.py index 44a15f55..2c866f46 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -719,6 +719,63 @@ def _lookup_dataset_specs_for_tasks(task_names: Iterable[str]) -> list[DatasetSp return specs +def _build_task_aux_map() -> dict[str, tuple[list[str], list[dict]]]: + """Map task name → (hf_models, hf_dataset_files) from all task groups.""" + data = _load_task_groups_data() + parsed = _parse_task_groups(list(data.get("task_groups", {}).keys())) + aux: dict[str, tuple[list[str], list[dict]]] = {} + for _, group in parsed.items(): + if isinstance(group, TaskGroup): + for t in group.tasks: + if t.name not in aux and (t.hf_models or t.hf_dataset_files): + aux[t.name] = (t.hf_models or [], t.hf_dataset_files or []) + return aux + + +def _lookup_hf_model_repos_for_tasks(task_names: Iterable[str]) -> list[str]: + """``hf_models`` pre-download repos for individual task names. + + Mirrors :func:`_collect_hf_model_repos` for tasks scheduled without a + task group (bare ``--tasks`` or a ``--check`` re-schedule CSV) — those + paths previously skipped auxiliary-model staging entirely, so contrib + rows failed on the air-gapped compute node. + """ + aux = _build_task_aux_map() + repos: list[str] = [] + seen: set[str] = set() + for name in task_names: + models, _files = aux.get(str(name).strip(), ([], [])) + for repo_id in models: + if repo_id not in seen: + seen.add(repo_id) + repos.append(repo_id) + return repos + + +def _lookup_hf_dataset_files_for_tasks(task_names: Iterable[str]) -> list[dict]: + """``hf_dataset_files`` specs for individual task names, merged per + (repo_id, revision) like :func:`_collect_hf_dataset_files`.""" + aux = _build_task_aux_map() + merged: dict[tuple[str, str | None], list[str]] = {} + for name in task_names: + _models, file_specs = aux.get(str(name).strip(), ([], [])) + for spec in file_specs: + repo_id = spec.get("repo_id", "") + if not repo_id: + continue + pats = merged.setdefault((repo_id, spec.get("revision")), []) + for p in spec.get("patterns") or []: + if p not in pats: + pats.append(p) + result = [] + for (rid, rev), pats in merged.items(): + entry: dict = {"repo_id": rid, "patterns": pats} + if rev: + entry["revision"] = rev + result.append(entry) + return result + + def _build_task_suite_map() -> dict[str, str]: """Build a mapping from task names to their suite from all task groups.""" data = _load_task_groups_data() diff --git a/oellm/utils.py b/oellm/utils.py index 5d9ef2eb..67a5eb4b 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -151,11 +151,18 @@ class _Default(dict): def __missing__(self, key): return "{" + key + "}" - base_ctx = _Default({**os.environ, **{k: str(v) for k, v in cluster_cfg_raw.items()}}) + # Template-resolution precedence: user environment > cluster values > + # shared values. The user env must win INSIDE templated values too — + # ``export EVAL_BASE_DIR=/custom`` has to propagate into + # ``EVAL_OUTPUT_DIR: "{EVAL_BASE_DIR}/{USER}"``, not just override the + # variable itself (the export below is setdefault, so exported vars also + # win at that level). + cluster_str = {k: str(v) for k, v in cluster_cfg_raw.items()} + base_ctx = _Default({**cluster_str, **os.environ}) resolved_shared = {k: str(v).format_map(base_ctx) for k, v in shared_cfg.items()} - ctx = _Default({**base_ctx, **resolved_shared}) + ctx = _Default({**resolved_shared, **cluster_str, **os.environ}) resolved_cluster = {k: str(v).format_map(ctx) for k, v in cluster_cfg_raw.items()} @@ -268,15 +275,20 @@ def _expand_local_model_paths(model: str | Path) -> list[Path]: return model_paths -def _process_model_paths(models: Iterable[str]): - """ - Processes model strings into a dict of model paths. +def _process_model_paths(models: Iterable[str]) -> dict[str, str | None]: + """Pre-download hub models and return their resolved revisions. Each model string can be a local path or a huggingface model identifier. - This function expands directory paths that contain multiple checkpoints. + Local directory paths that contain multiple checkpoints are expanded. + Returns ``{model: resolved_commit_or_None}`` for hub models — the commit + actually present in the shared cache, recorded into the run's + ``provenance.json``. Models are otherwise unpinned: whatever revision is + cached is what the offline compute node evaluates, so recording it is the + only way a collected number stays traceable. """ from huggingface_hub import snapshot_download + revisions: dict[str, str | None] = {} console = get_console() models_list = list(models) @@ -312,13 +324,18 @@ def _process_model_paths(models: Iterable[str]): status.update(f"Downloading '{repo_id}' ({idx}/{len(models_list)})") try: - snapshot_download( + _local = snapshot_download( repo_id=repo_id, cache_dir=Path(os.getenv("HF_HOME")) / "hub" if "HF_HOME" in os.environ else None, **snapshot_kwargs, ) + revisions[str(model)] = ( + Path(_local).name + if Path(_local).parent.name == "snapshots" + else None + ) per_model_paths.append(model) except Exception as e: logging.warning( @@ -336,10 +353,15 @@ def _process_model_paths(models: Iterable[str]): status.update(f"Downloading '{model}' ({idx}/{len(models_list)})") # snapshot_download is idempotent — it skips files that # are already cached and only fetches missing ones. - snapshot_download( + _local = snapshot_download( repo_id=model, cache_dir=cache_dir, ) + revisions[str(model)] = ( + Path(_local).name + if Path(_local).parent.name == "snapshots" + else None + ) per_model_paths.append(model) if not per_model_paths: @@ -347,6 +369,8 @@ def _process_model_paths(models: Iterable[str]): f"Could not find any valid model for '{model}'. It will be skipped." ) + return revisions + def _pre_download_hf_model_repos(repo_ids: list[str]) -> None: """Download auxiliary HF model repos (e.g. SAM2) required by contrib suites.""" @@ -477,9 +501,13 @@ def _pre_download_datasets_from_specs( max_workers=2, ) except Exception as e: - logging.warning( - f"snapshot_download failed for '{rev_label}': {e}" - ) + # Media files are NOT fetched by load_dataset() below, + # so a failed snapshot means every row of this task + # fails hours later on the air-gapped compute node. + # Strict contract: aggregate and abort before SLURM + # submission (bypass with --skip-checks if the cache + # is already populated out-of-band). + failures.append((rev_label, e)) # Build the Arrow cache — this is what makes the dataset loadable on the # OFFLINE compute nodes. load_dataset() needs the BUILT dataset under diff --git a/pyproject.toml b/pyproject.toml index 69fd5428..fac1eaa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "oellm-eval" -version = "0.1.0" +version = "0.2.0" description = "OpenEuroLLM CLI" readme = "README.md" requires-python = ">=3.12,<3.13" @@ -80,7 +80,9 @@ audiobench = [ # in its own venv: ``uv pip install '.[evalchemy]'``. Pinned versions are # what evalchemy upstream requires; see docs/VENV.md. evalchemy = [ - "lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@etashg/tokenize_fix", + # Pinned to the tip of the etashg/tokenize_fix branch as of 2026-07-03 — + # a mutable branch ref made evaluated engine code drift over time. + "lm-eval @ git+https://github.com/EtashGuha/lm-evaluation-harness@4071e1823bc1d05d1b394e03925b436a46de637e", "scipy==1.17.0", "datasets==3.6.0", "transformers==4.57.6", diff --git a/scripts/pivot_results.py b/scripts/pivot_results.py index 91058d2d..109e3c04 100644 --- a/scripts/pivot_results.py +++ b/scripts/pivot_results.py @@ -6,14 +6,25 @@ def main() -> None: parser = argparse.ArgumentParser( - description="Pivot eval CSVs into a model × task leaderboard." + description=( + "Pivot eval CSVs into a model × task leaderboard. Cell values are " + "the 0-100 normalized scores (performance_normalized) when " + "available; rows without one fall back to the raw performance and " + "their column is labelled ', raw'." + ) ) parser.add_argument("csvs", nargs="+", help="One or more result CSV files") parser.add_argument( "-o", "--output", default="leaderboard.csv", help="Output CSV path" ) parser.add_argument( - "--no-average", action="store_true", help="Do not append an average column" + "--average", + action="store_true", + help=( + "Append a mean over the NORMALIZED columns. Caveat: the mean " + "skips NaNs, so models missing tasks average over different " + "subsets — only comparable when every model covers every column." + ), ) args = parser.parse_args() @@ -36,17 +47,42 @@ def main() -> None: print(f"Error: input CSVs missing columns: {missing}", file=sys.stderr) sys.exit(1) - df["task_label"] = df.apply( - lambda r: f"{r['task']} ({int(r['n_shot'])}-shot)", axis=1 - ) - df = df.drop_duplicates(subset=["model_name", "task_label"], keep="last") + if "metric_name" not in df.columns: + df["metric_name"] = "" + has_norm = "performance_normalized" in df.columns + + # Metric identity is part of the column label so rows that legitimately + # differ only in metric_name don't collapse into one cell; the n_shot + # label is not int()-coerced because collect legitimately emits "unknown". + def _label(r) -> str: + metric = str(r["metric_name"]).split(",")[0] or "metric" + norm_ok = has_norm and pd.notna(r.get("performance_normalized")) + suffix = "" if norm_ok else ", raw" + return f"{r['task']} ({r['n_shot']}-shot, {metric}{suffix})" - pivot = df.pivot(index="model_name", columns="task_label", values="performance") + df["task_label"] = df.apply(_label, axis=1) + if has_norm: + df["value"] = df["performance_normalized"].where( + df["performance_normalized"].notna(), df["performance"] + ) + else: + df["value"] = df["performance"] + + # keep="last" matches collect's newest-wins duplicate semantics. + df = df.drop_duplicates(subset=["model_name", "task_label"], keep="last") + pivot = df.pivot(index="model_name", columns="task_label", values="value") pivot = pivot[sorted(pivot.columns)] - if not args.no_average: - pivot["average"] = pivot.mean(axis=1) + if args.average: + norm_cols = [c for c in pivot.columns if ", raw" not in c] + if norm_cols: + pivot["average (normalized cols)"] = pivot[norm_cols].mean(axis=1) + print( + "Note: the average covers normalized columns only and skips " + "NaNs — models missing tasks average over different subsets.", + file=sys.stderr, + ) pivot = pivot.reset_index() pivot.to_csv(args.output, index=False) diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 0ed3c1fd..26ed5b15 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -438,7 +438,7 @@ def test_json_file_written_alongside_csv(self, tmp_path): json_path = tmp_path / "out.json" assert json_path.exists() envelope = json.loads(json_path.read_text()) - assert envelope["version"] == "1.1" + assert envelope["version"] == "1.2" assert len(envelope["results"]) == 1 record = envelope["results"][0] assert record["task"] == "copa" diff --git a/tests/test_collection_and_scheduling.py b/tests/test_collection_and_scheduling.py new file mode 100644 index 00000000..a218bd5d --- /dev/null +++ b/tests/test_collection_and_scheduling.py @@ -0,0 +1,412 @@ +"""Regression tests for collection correctness, provenance, and scheduling +recovery: duplicate-result resolution, silent group/prefix omissions, the +lighteval round-trip, per-task scale overrides, model-aware compare, +cluster-env precedence, eval_suite normalisation, task-scope aux staging, +and the v1.2 provenance envelope. +""" + +import csv +import json +import os +import time +from pathlib import Path + +import pytest + +from oellm import __version__ +from oellm.results import _normalize_to_100, collect_results + + +def _write(results_dir: Path, name: str, payload: dict, mtime: float | None = None): + p = results_dir / name + p.write_text(json.dumps(payload)) + if mtime is not None: + os.utime(p, (mtime, mtime)) + return p + + +def _rows(output_csv: str) -> list[dict]: + with open(output_csv) as f: + return list(csv.DictReader(f)) + + +M = {"model_name_or_path": "m"} + + +# ── duplicate results resolve to the newest file, loudly ──────────────── + + +class TestDuplicateResolution: + def test_newest_mtime_wins_regardless_of_name_order(self, tmp_path, capsys): + d = tmp_path / "results" + d.mkdir() + now = time.time() + # "zz" sorts after "aa" by name; mtime must decide, not the name. + _write( + d, + "zz_stale.json", + {**M, "results": {"boolq": {"acc,none": 0.10}}, "n-shot": {"boolq": 0}}, + mtime=now - 100, + ) + _write( + d, + "aa_fresh.json", + {**M, "results": {"boolq": {"acc,none": 0.85}}, "n-shot": {"boolq": 0}}, + mtime=now, + ) + out = str(tmp_path / "out.csv") + collect_results(str(tmp_path), out) + rows = _rows(out) + assert len(rows) == 1 + assert float(rows[0]["performance"]) == pytest.approx(0.85) + # rich wraps log lines; compare on whitespace-normalized output + assert "Duplicate results" in " ".join(capsys.readouterr().out.split()) + + +# ── group/prefix omission mechanisms ──────────────────────────────────── + + +class TestGroupCollection: + def _collect(self, tmp_path, payload, caplog=None): + d = tmp_path / "results" + d.mkdir() + _write(d, "r.json", payload) + out = str(tmp_path / "out.csv") + collect_results(str(tmp_path), out) + return _rows(out) if Path(out).exists() else [] + + def test_all_top_level_groups_collected(self, tmp_path): + rows = self._collect( + tmp_path, + { + **M, + "results": { + "g1": {"acc,none": 0.5}, + "g2": {"acc,none": 0.6}, + "s1": {"acc,none": 0.5}, + "s2": {"acc,none": 0.6}, + }, + "groups": {"g1": {"acc,none": 0.5}, "g2": {"acc,none": 0.6}}, + "group_subtasks": {"g1": ["s1"], "g2": ["s2"]}, + "n-shot": {"g1": 0, "g2": 0}, + }, + ) + assert {r["task"] for r in rows} == {"g1", "g2"} + + def test_metricless_parent_falls_through_to_child_groups(self, tmp_path): + rows = self._collect( + tmp_path, + { + **M, + "results": { + "parent": {}, + "childg": {"acc,none": 0.6}, + "s1": {"acc,none": 0.6}, + }, + "groups": {"parent": {}, "childg": {"acc,none": 0.6}}, + "group_subtasks": {"parent": ["childg"], "childg": ["s1"]}, + "n-shot": {"childg": 0}, + }, + ) + assert [r["task"] for r in rows] == ["childg"] + + def test_standalone_task_in_group_file_is_kept(self, tmp_path): + rows = self._collect( + tmp_path, + { + **M, + "results": { + "g1": {"acc,none": 0.5}, + "s1": {"acc,none": 0.5}, + "lone": {"acc,none": 0.7}, + }, + "groups": {"g1": {"acc,none": 0.5}}, + "group_subtasks": {"g1": ["s1"]}, + "n-shot": {"g1": 0, "lone": 0}, + }, + ) + assert {r["task"] for r in rows} == {"g1", "lone"} + + def test_mmlu_pro_is_not_eaten_by_prefix_rule(self, tmp_path): + rows = self._collect( + tmp_path, + { + **M, + "results": {"mmlu_pro": {"acc,none": 0.4}}, + "n-shot": {"mmlu_pro": 5}, + }, + ) + assert [(r["task"], r["n_shot"]) for r in rows] == [("mmlu_pro", "5")] + + def test_mmlu_subtasks_still_skipped_when_aggregate_present(self, tmp_path): + rows = self._collect( + tmp_path, + { + **M, + "results": { + "mmlu": {"acc,none": 0.5}, + "mmlu_abstract_algebra": {"acc,none": 0.3}, + }, + "n-shot": {"mmlu": 5}, + }, + ) + assert [r["task"] for r in rows] == ["mmlu"] + + def test_global_mmlu_underscore_language_aggregate_kept(self, tmp_path): + rows = self._collect( + tmp_path, + { + **M, + "results": { + "global_mmlu_full_zh_hans": {"acc,none": 0.3}, + "global_mmlu_full_zh_hans_anatomy": {"acc,none": 0.31}, + }, + "n-shot": {"global_mmlu_full_zh_hans": 0}, + }, + ) + assert [r["task"] for r in rows] == ["global_mmlu_full_zh_hans"] + + def test_partial_children_aggregate_warns(self, tmp_path, capsys): + d = tmp_path / "results" + d.mkdir() + _write( + d, + "r.json", + { + **M, + "results": {"par": {"alias": " "}, "c1": {"acc,none": 0.5}}, + "group_subtasks": {"par": ["c1", "c2"]}, + "configs": {"c1": {"num_fewshot": 0}}, + }, + ) + out = str(tmp_path / "out.csv") + collect_results(str(tmp_path), out) + assert "1/2 subtasks present" in " ".join(capsys.readouterr().out.split()) + + def test_zero_row_file_warns(self, tmp_path, capsys): + d = tmp_path / "results" + d.mkdir() + _write(d, "r.json", {**M, "results": {"ghost": {"alias": " "}}}) + collect_results(str(tmp_path), str(tmp_path / "out.csv")) + assert "contributed zero rows" in " ".join(capsys.readouterr().out.split()) + + +# ── lighteval-shaped round-trip through collect + --check ─────── + + +class TestLightevalRoundTrip: + PAYLOAD = { + "config_general": {"model_name": "m"}, + "results": { + "all": {"acc_norm": 0.61}, + "belebele_fra_Latn_cf|5": {"acc_norm": 0.61, "acc_norm_stderr": 0.01}, + }, + } + + def test_task_and_nshot_recovered(self, tmp_path): + d = tmp_path / "results" + d.mkdir() + _write(d, "r.json", self.PAYLOAD) + out = str(tmp_path / "out.csv") + collect_results(str(tmp_path), out) + rows = _rows(out) + assert [(r["task"], r["n_shot"]) for r in rows] == [("belebele_fra_Latn_cf", "5")] + + def test_check_marks_lighteval_job_complete(self, tmp_path): + d = tmp_path / "results" + d.mkdir() + _write(d, "r.json", self.PAYLOAD) + (tmp_path / "jobs.csv").write_text( + "model_path,task_path,n_shot,eval_suite\nm,belebele_fra_Latn_cf,5,lighteval\n" + ) + out = tmp_path / "out.csv" + collect_results(str(tmp_path), str(out), check=True) + missing = out.with_name("out_missing.csv") + assert not missing.exists(), "completed lighteval job was re-reported as missing" + + +# ── per-task scale overrides ──────────────────────────────────────────── + + +class TestScaleOverrides: + def test_squadv2_f1_is_percentage_scale(self): + assert _normalize_to_100(55.3, "f1,none", "squadv2") == pytest.approx(55.3) + + def test_coqa_f1_keeps_fraction_scale(self): + assert _normalize_to_100(0.553, "f1,none", "coqa") == pytest.approx(55.3) + + def test_voicebench_judge_is_likert_scale(self): + assert _normalize_to_100( + 3.4, "llm_as_judge_eval,none", "voicebench_commoneval" + ) == pytest.approx(68.0) + + +# ── Envelope v1.2 + provenance sidecar ─────────────────────────────────────── + + +class TestProvenanceEnvelope: + def test_sidecar_embedded_and_namespace_reserved(self, tmp_path): + d = tmp_path / "results" + d.mkdir() + _write( + d, + "r.json", + {**M, "results": {"copa": {"acc,none": 0.8}}, "n-shot": {"copa": 0}}, + ) + (tmp_path / "provenance.json").write_text( + json.dumps({"schema": 1, "model_revisions": {"m": "abc123"}}) + ) + out = tmp_path / "out.csv" + collect_results(str(tmp_path), str(out)) + envelope = json.loads((tmp_path / "out.json").read_text()) + assert envelope["version"] == "1.2" + assert envelope["metadata"] == {} + assert envelope["oellm_version"] == __version__ + assert envelope["runs"][0]["model_revisions"] == {"m": "abc123"} + + +# ── eval_suite normalisation in the runner ─────────────────────────────── + + +class TestSuiteNormalisation: + @pytest.fixture(autouse=True) + def _fake_adapter(self, monkeypatch): + monkeypatch.setattr("oellm.runner.detect_lmms_model_type", lambda p: "llava_hf") + + @pytest.mark.parametrize( + "raw", ["LMMS_EVAL", " lmms_eval ", "Lmms-Eval", "lmms_eval"] + ) + def test_noncanonical_spellings_resolve(self, raw): + from oellm.constants import EvaluationJob + from oellm.runner import EvalRunner + + job = EvaluationJob(model_path="x", task_path="t", n_shot=0, eval_suite=raw) + assert EvalRunner().resolve_suite(job) == "lmms_eval:llava_hf" + + def test_already_suffixed_row_is_idempotent(self): + from oellm.constants import EvaluationJob + from oellm.runner import EvalRunner + + job = EvaluationJob( + model_path="x", task_path="t", n_shot=0, eval_suite="lmms_eval:qwen2_vl" + ) + assert EvalRunner().resolve_suite(job) == "lmms_eval:qwen2_vl" + + +# ── user env overrides propagate into templated cluster vars ──────────── + + +class TestClusterEnvPrecedence: + def test_eval_base_dir_override_reaches_output_dir(self, monkeypatch): + from oellm import utils + + monkeypatch.setattr(utils.socket, "gethostname", lambda: "login01.leonardo.local") + for var in ( + "EVAL_OUTPUT_DIR", + "PARTITION", + "ACCOUNT", + "GPUS_PER_NODE", + "TIME_LIMIT", + ): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("USER", "testuser") + monkeypatch.setenv("EVAL_BASE_DIR", "/custom/base") + # HF_HOME must come from user env: it is a required var on every + # cluster but the stock leonardo entry does not define it — the test + # must not depend on local clusters.yaml additions. + monkeypatch.setenv("HF_HOME", "/custom/hf") + utils._load_cluster_env() + assert os.environ["EVAL_OUTPUT_DIR"] == "/custom/base/testuser" + + +# ── aux staging resolvable from bare task names ───────────────────────── + + +class TestTaskScopedAuxStaging: + def test_hf_models_found_without_group(self): + from oellm.task_groups import _lookup_hf_model_repos_for_tasks + + repos = _lookup_hf_model_repos_for_tasks(["regiondial_refcocog"]) + assert "Ricky06662/TaskRouter-1.5B" in repos + assert "facebook/sam2-hiera-large" in repos + + def test_hf_dataset_files_found_without_group(self): + from oellm.task_groups import _lookup_hf_dataset_files_for_tasks + + specs = _lookup_hf_dataset_files_for_tasks(["regiondial_refcocog"]) + assert any( + s["repo_id"] == "lmsdss/regionreasoner_test_data" and s["patterns"] + for s in specs + ) + + def test_unknown_tasks_yield_nothing(self): + from oellm.task_groups import ( + _lookup_hf_dataset_files_for_tasks, + _lookup_hf_model_repos_for_tasks, + ) + + assert _lookup_hf_model_repos_for_tasks(["hellaswag"]) == [] + assert _lookup_hf_dataset_files_for_tasks(["hellaswag"]) == [] + + +# ── compare is model-aware and reads collect's own output ──────────────── + + +class TestCompare: + def _envelope(self, rows): + return json.dumps({"version": "1.2", "results": rows}) + + def test_multi_model_rows_both_shown_and_mixed_nshot_sorts(self, tmp_path, capsys): + from oellm.main import compare + + rows_a = [ + { + "model": "modelA", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.5, + }, + { + "model": "modelB", + "task": "mmlu", + "n_shot": 5, + "metric": "acc", + "performance": 0.6, + }, + { + "model": "modelA", + "task": "mvbench", + "n_shot": "unknown", + "metric": "mvbench_accuracy", + "performance": 51.0, + }, + ] + a = tmp_path / "a.json" + b = tmp_path / "b.json" + a.write_text(self._envelope(rows_a)) + b.write_text(self._envelope(rows_a)) + compare(str(a), str(b)) # must not raise (mixed n_shot types) + out = capsys.readouterr().out + assert "modelA" in out and "modelB" in out + + def test_directory_mode_reads_eval_results_json(self, tmp_path, capsys): + from oellm.main import compare + + run = tmp_path / "run" + run.mkdir() + (run / "eval_results.json").write_text( + self._envelope( + [ + { + "model": "m", + "task": "copa", + "n_shot": 0, + "metric": "acc", + "performance": 0.8, + } + ] + ) + ) + compare(str(run), str(run)) + assert "copa" in capsys.readouterr().out diff --git a/tests/test_metric_snapshots.py b/tests/test_metric_snapshots.py index 1e1c889b..0a6ac045 100644 --- a/tests/test_metric_snapshots.py +++ b/tests/test_metric_snapshots.py @@ -93,6 +93,46 @@ "longvideobench_val_v": { "longvideobench_val_v/lvb_acc,none": 0.473, }, + # MME emits raw point sums (cognition /800 preferred per task_metrics). + "mme": { + "mme/mme_cognition_score,none": 512.3, + "mme/mme_perception_score,none": 1373.2, + }, + # ── Newer image benchmarks (added 41eabb8) ── + "ocrbench_v2": { + "ocrbench_v2/ocrbench_v2_accuracy,none": 0.412, + }, + "realworldqa": { + "realworldqa/exact_match,none": 0.583, + }, + # NOTE: emitted scale not yet confirmed against a real run — the fixture + # pins map↔scale consistency (0–1 assumed). + "mmerealworld": { + "mmerealworld/mme_realworld_score,none": 0.437, + }, + "mmstar": { + "mmstar/average,none": 0.451, + }, + "ai2d": { + "ai2d/exact_match,none": 0.702, + }, + # NOTE: emitted scale not yet confirmed against a real run (0–100 assumed). + "mathvision_test": { + "mathvision_test/mathvision_standard_eval,none": 19.2, + }, + "seedbench": { + "seedbench/seed_image,none": 0.612, + }, + # ── Per-task scale overrides (TASK_METRIC_SCALE_OVERRIDES) ── + # lm-eval squadv2 wraps the official squad_v2 metric → 0–100 percentages. + "squadv2": { + "f1,none": 55.3, + "exact,none": 50.1, + }, + # lmms-eval voicebench judge returns the raw 1–5 average. + "voicebench_commoneval": { + "voicebench_commoneval/llm_as_judge_eval,none": 3.4, + }, } @@ -144,28 +184,21 @@ def test_every_snapshot_has_a_task_metrics_entry(task_metrics: dict) -> None: ) -# List the image+video tasks that MUST have a snapshot. Sourced from the -# image-vqa, video-understanding task groups. -REQUIRED_TASKS_WITH_SNAPSHOT: set[str] = { - "vqav2_val", - "mmbench_en_dev", - "mmmu_val", - "chartqa", - "docvqa_val", - "textvqa_val", - "ocrbench", - "mathvista_testmini_cot", - "mathvista_testmini_format", - "mathvista_testmini_solution", - "video_mmmu_perception", - "video_mmmu_comprehension", - "video_mmmu_adaptation", - "mvbench", - "egoschema_subset", - "videomme", - "activitynetqa", - "longvideobench_val_v", -} +# Every task of every image-* / video-* task group MUST have a snapshot — +# derived from task-groups.yaml so the set cannot drift when benchmarks are +# added. +def _image_video_tasks_from_yaml() -> set[str]: + data = yaml.safe_load((files("oellm.resources") / "task-groups.yaml").read_text()) + tasks: set[str] = set() + for gname, g in data.get("task_groups", {}).items(): + if gname.startswith(("image-", "video-")): + for t in g.get("tasks", []): + if t.get("task"): + tasks.add(t["task"]) + return tasks + + +REQUIRED_TASKS_WITH_SNAPSHOT: set[str] = _image_video_tasks_from_yaml() def test_all_required_image_video_tasks_have_snapshot() -> None: @@ -193,7 +226,7 @@ def test_normalized_value_is_in_zero_to_hundred_range( from oellm.results import _normalize_to_100 value, resolved_key = _resolve_metric(task_name, SNAPSHOTS[task_name], task_metrics) - normalized = _normalize_to_100(value, resolved_key) + normalized = _normalize_to_100(value, resolved_key, task_name) assert normalized is not None, ( f"_normalize_to_100 returned None for {task_name} " f"(value={value}, key={resolved_key}). The metric's native scale is " diff --git a/tests/test_quantization_and_timeout.py b/tests/test_quantization_and_timeout.py new file mode 100644 index 00000000..8584986d --- /dev/null +++ b/tests/test_quantization_and_timeout.py @@ -0,0 +1,179 @@ +"""Tests for the bitsandbytes quantization flags and the per-row timeout guard.""" + +import json +from unittest.mock import patch + +import pytest + +from oellm.config import EvalConfig + + +class TestQuantizationConfig: + def test_yaml_roundtrip(self, tmp_path): + p = tmp_path / "cfg.yaml" + p.write_text("models: [m]\ntask_groups: [image-vqa]\nload_in_4bit: true\n") + cfg = EvalConfig.from_yaml(p) + assert cfg.load_in_4bit is True + assert cfg.load_in_8bit is False + + def test_cli_overrides_yaml(self, tmp_path): + p = tmp_path / "cfg.yaml" + p.write_text("models: [m]\ntask_groups: [g]\nload_in_4bit: true\n") + yaml_cfg = EvalConfig.from_yaml(p) + cli_cfg = EvalConfig.from_cli_kwargs(load_in_4bit=False) + merged = yaml_cfg.merge(cli_cfg) + assert merged.load_in_4bit is False + + def test_mutual_exclusion(self): + cfg = EvalConfig( + models=["m"], + tasks=["t"], + n_shot=[0], + load_in_4bit=True, + load_in_8bit=True, + ) + with pytest.raises(ValueError, match="mutually exclusive"): + cfg.validate() + + +def _schedule(tmp_path, monkeypatch, **kw): + from oellm.scheduler import schedule_evals + + monkeypatch.setenv("EVAL_OUTPUT_DIR", str(tmp_path)) + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + ): + schedule_evals(dry_run=True, skip_checks=True, **kw) + sbatch = next(tmp_path.glob("**/submit_evals.sbatch")).read_text() + prov = json.loads(next(tmp_path.glob("**/provenance.json")).read_text()) + return sbatch, prov + + +class TestQuantizationScheduling: + def test_4bit_reaches_engines_and_provenance(self, tmp_path, monkeypatch): + sbatch, prov = _schedule( + tmp_path, + monkeypatch, + models="org/m", + tasks="hellaswag", + n_shot=0, + load_in_4bit=True, + ) + # lm_eval + lmms_eval + evalchemy model_args each carry the flag + assert sbatch.count(",load_in_4bit=True") == 3 + assert 'export OELLM_QUANTIZATION="4bit"' in sbatch + assert prov["quantization"] == "4bit" + + def test_default_is_full_precision(self, tmp_path, monkeypatch): + sbatch, prov = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert "load_in_4bit" not in sbatch + assert "load_in_8bit" not in sbatch + assert 'export OELLM_QUANTIZATION=""' in sbatch + assert prov["quantization"] is None + + def test_both_flags_rejected(self, tmp_path, monkeypatch): + with pytest.raises(ValueError, match="mutually exclusive"): + _schedule( + tmp_path, + monkeypatch, + models="org/m", + tasks="t", + n_shot=0, + load_in_4bit=True, + load_in_8bit=True, + ) + + +class TestRowTimeout: + def test_helper_present_and_wraps_engines(self, tmp_path, monkeypatch): + monkeypatch.delenv("ROW_TIMEOUT", raising=False) + sbatch, prov = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert "_maybe_timeout()" in sbatch # helper defined + # wraps: run_python (venv + container), lighteval (venv + container), + # evalchemy + assert sbatch.count("_maybe_timeout ") >= 5 + assert prov["row_timeout"] is None + + def test_serial_slice_without_timeout_warns(self, tmp_path, monkeypatch, capsys): + monkeypatch.setenv("QUEUE_LIMIT", "1") + monkeypatch.delenv("ROW_TIMEOUT", raising=False) + _schedule( + tmp_path, + monkeypatch, + models="org/m", + tasks="hellaswag,winogrande", + n_shot=0, + ) + out = " ".join(capsys.readouterr().out.split()) + assert "no per-row timeout" in out + + def test_row_timeout_recorded_in_provenance(self, tmp_path, monkeypatch): + monkeypatch.setenv("ROW_TIMEOUT", "2h") + _, prov = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert prov["row_timeout"] == "2h" + + +class TestEngineVersionProvenance: + def test_versions_probed_from_venv(self, tmp_path, monkeypatch): + from oellm.scheduler import schedule_evals + + probed = [] + + def fake_probe(python_bin, module): + probed.append(module) + # lighteval "missing" in this venv — must be omitted, not error + return (module != "lighteval", "9.9.9") + + monkeypatch.setattr("oellm.envcheck.probe_import", fake_probe) + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + (venv / "bin" / "python").write_text("") + monkeypatch.setenv("EVAL_OUTPUT_DIR", str(tmp_path)) + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._process_model_paths", return_value={}), + patch("oellm.scheduler._lookup_dataset_specs_for_tasks", return_value=[]), + patch("oellm.envcheck.check_scheduled_environment"), + patch("oellm.scheduler.check_scheduled_environment", create=True), + ): + import oellm.envcheck as envcheck_mod + + monkeypatch.setattr( + envcheck_mod, "check_scheduled_environment", lambda *a, **k: None + ) + schedule_evals( + models="org/m", + tasks="hellaswag", + n_shot=0, + dry_run=True, + venv_path=str(venv), + ) + prov = json.loads(next(tmp_path.glob("**/provenance.json")).read_text()) + assert prov["engine_versions"] == { + "lm_eval": "9.9.9", + "lmms_eval": "9.9.9", + "transformers": "9.9.9", + "torch": "9.9.9", + } + assert prov["container_image"] is None + assert set(probed) == { + "lm_eval", + "lighteval", + "lmms_eval", + "transformers", + "torch", + } + + def test_skip_checks_skips_probing(self, tmp_path, monkeypatch): + sbatch, prov = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert prov["engine_versions"] == {} diff --git a/tests/test_reporter.py b/tests/test_reporter.py index 0b0de8d2..81f08c23 100644 --- a/tests/test_reporter.py +++ b/tests/test_reporter.py @@ -32,7 +32,7 @@ def test_write_json_schema_version(tmp_path: Path) -> None: out = tmp_path / "results.json" write_results_json(_SAMPLE_ROWS, out) data = json.loads(out.read_text()) - assert data["version"] == SCHEMA_VERSION == "1.1" + assert data["version"] == SCHEMA_VERSION == "1.2" def test_write_json_result_fields(tmp_path: Path) -> None: @@ -77,7 +77,7 @@ def test_write_json_empty_rows(tmp_path: Path) -> None: write_results_json([], out) data = json.loads(out.read_text()) assert data["results"] == [] - assert data["version"] == "1.1" + assert data["version"] == "1.2" def test_write_json_creates_parent_dirs(tmp_path: Path) -> None: diff --git a/tests/test_utils.py b/tests/test_utils.py index a6c1b2c0..1cdfc762 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -151,10 +151,10 @@ def fake_load_dataset(repo_id, **kwargs): _pre_download_datasets_from_specs(specs) assert calls == ["org/dataset-a", "org/dataset-b"] - def test_snapshot_failure_alone_does_not_raise_if_load_dataset_works( - self, monkeypatch - ): - """snapshot_download is best-effort; load_dataset success is enough.""" + def test_snapshot_failure_raises_even_if_load_dataset_works(self, monkeypatch): + """Media snapshot failures are STRICT: load_dataset alone does not + stage the separate media files (video/audio assets), so proceeding + would schedule rows that fail hours later on the air-gapped node.""" def boom_snapshot(*args, **kwargs): raise OSError("simulated snapshot HTTP 429") @@ -168,8 +168,8 @@ def fake_load_dataset(*args, **kwargs): specs = [_FakeSpec(repo_id="org/dataset-a", needs_snapshot_download=True)] - # snapshot_download fails but load_dataset succeeds → no raise. - _pre_download_datasets_from_specs(specs) + with pytest.raises(RuntimeError, match="Pre-download failed"): + _pre_download_datasets_from_specs(specs) class TestPreDownloadRevisions: From b463d0480c0d1e6094255adab9f11e63c57a614b Mon Sep 17 00:00:00 2001 From: islobozhan Date: Wed, 22 Jul 2026 10:06:53 +0200 Subject: [PATCH 30/44] [Base] Phase 4 tabular and time series support --- oellm/constants.py | 19 +- oellm/contrib/CONTRIBUTING.md | 79 +++++- oellm/contrib/regiondial_bench/metrics.py | 74 +++-- oellm/contrib/regiondial_bench/suite.py | 14 +- oellm/core/__init__.py | 22 +- oellm/core/base_metric.py | 52 ++-- oellm/core/base_model_adapter.py | 50 +++- oellm/core/base_task.py | 8 +- oellm/main.py | 5 +- oellm/registry.py | 21 ++ .../custom_lm_eval_tasks/tabfact/tabfact.yaml | 24 ++ .../custom_lm_eval_tasks/tabfact/utils.py | 35 +++ .../timeseriesexam/timeseriesexam.yaml | 21 ++ .../timeseriesexam/utils.py | 64 +++++ oellm/resources/task-groups.yaml | 21 ++ oellm/resources/template.sbatch | 14 +- oellm/results.py | 53 ++++ oellm/scheduler.py | 37 ++- tests/test_base_interfaces.py | 33 ++- tests/test_plugin_protocol.py | 258 ++++++++++++++++++ tests/test_quantization_and_timeout.py | 38 +++ tests/test_regiondial_bench.py | 41 ++- 22 files changed, 851 insertions(+), 132 deletions(-) create mode 100644 oellm/resources/custom_lm_eval_tasks/tabfact/tabfact.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/tabfact/utils.py create mode 100644 oellm/resources/custom_lm_eval_tasks/timeseriesexam/timeseriesexam.yaml create mode 100644 oellm/resources/custom_lm_eval_tasks/timeseriesexam/utils.py create mode 100644 tests/test_plugin_protocol.py diff --git a/oellm/constants.py b/oellm/constants.py index 44d690cb..ad70a2a9 100644 --- a/oellm/constants.py +++ b/oellm/constants.py @@ -88,15 +88,26 @@ def detect_lmms_model_type(model_path: str) -> str: lmms-eval requires --model (e.g. llava_hf, qwen2_5_vl). This is inferred from the model name so users never need to set it manually. - To add support for a new model family, add an entry to LMMS_MODEL_ADAPTERS - above or register a BaseModelAdapter via the contrib plugin system. + To add support for a new model family, either add an entry to + LMMS_MODEL_ADAPTERS above, or export ``LMMS_MODEL_ADAPTERS`` (same shape) + from a contrib suite module — contrib entries are consulted FIRST, so + plugins can route new families without touching core files. """ name = str(model_path).lower() + + # Contrib-registered patterns take precedence. Lazy import: the registry + # walks contrib packages; constants must stay import-light. + from oellm.registry import get_lmms_adapter_overrides # noqa: PLC0415 + + for patterns, adapter in get_lmms_adapter_overrides(): + if any(p in name for p in patterns): + return adapter + for patterns, adapter in LMMS_MODEL_ADAPTERS: if any(p in name for p in patterns): return adapter raise ValueError( f"Cannot auto-detect lmms-eval adapter class from model path '{model_path}'. " - "Add a pattern to LMMS_MODEL_ADAPTERS in constants.py or register a " - "BaseModelAdapter via a contrib plugin." + "Add a pattern to LMMS_MODEL_ADAPTERS in constants.py or export " + "LMMS_MODEL_ADAPTERS from a contrib suite module." ) diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index 77739375..eccb4abb 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -1,6 +1,7 @@ # Integrating Custom Benchmarks -There are two ways to add a benchmark to elliot-cli. +There are three ways to add a benchmark to elliot-cli, in increasing order +of effort. Pick the first one that fits. --- @@ -39,7 +40,51 @@ Supported `suite` values: --- -## Path 2 — Custom benchmark (new contrib suite) +## Path 2 — Custom lm-eval task (YAML + helpers, no plugin) + +Use this when the benchmark is **not** in any engine yet but fits lm-eval's +task model: an HF dataset in, a per-sample prompt and target out. You write +only the task definition; scheduling, staging, collection, and reporting are +the platform's existing lm-eval machinery. No core file changes. + +1. Create `oellm/resources/custom_lm_eval_tasks//.yaml` — a + standard lm-eval task config (plain `.yaml` directly in + `custom_lm_eval_tasks/` also works for simple tasks). Prompt logic that + doesn't fit YAML goes in a sibling `utils.py`, referenced as + `!function utils.my_fn`. The bundled directory is passed to lm_eval via + `--include_path` automatically; the `lm_eval_include_path` key in a run + config (`oellm-eval eval --config …`) can point at an out-of-tree + directory instead. +2. Wire `oellm/resources/task-groups.yaml`: a `task_metrics:` entry naming + the headline metric, and a task group with `suite: lm-eval-harness`. The + YAML `task:` field, the group's `task:` entry, and the `task_metrics:` + key must all match. +3. Add a conformance test class in `tests/test_plugin_protocol.py`: group + expansion, metric mapping, and prompt construction on a synthetic doc. + +Live references, in increasing order of trickiness: + +- `jeopardy.yaml`, `sib200/`, `arc_mt/` — plain YAML tasks. +- `tabfact/` — `utils.py` serializes tables into the prompt with a row cap; + few-shot drawn from the train split. +- `timeseriesexam/` — the full toolbox: `doc_to_choice`/`doc_to_target` as + functions (the dataset stores answers as option *text* with variable + option counts), fixed-policy subsampling of 1000+-point series, and + 0-shot only because the dataset ships just a test split. + +Gotchas: + +- Few-shot needs a split to draw from (`fewshot_split`); test-only datasets + must schedule with `n_shots: [0]`. +- Script-based datasets need `dataset_kwargs: {trust_remote_code: true}` in + the task YAML; parquet-native datasets need nothing. +- Any serialization policy (row caps, series subsampling) must be fixed and + documented in the task — every model has to see the identical prompt or + scores stop being comparable. + +--- + +## Path 3 — Custom benchmark (new contrib suite) Use this when the benchmark has its own inference script, custom metrics, or requires multi-GPU sharding. @@ -211,12 +256,11 @@ def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: pre-flight on the login node (before SLURM submission) and by `dispatch.py` on the compute node before `run()` is called. -> **Note on `parse_results`:** the protocol requires it (and the registry -> tests enforce it), but result collection currently parses the -> lmms-eval-shaped JSON that `run()` writes *generically* — `parse_results` -> is not invoked by `collect`. Treat the JSON shape written by `run()` as the -> real contract; keep `parse_results` correct so the suite is ready for -> format-specific collection. +> **Note on `parse_results`:** `oellm-eval collect` calls every suite's +> `parse_results` **first-chance** on each result JSON before falling back +> to the generic lmms-eval-shaped parsing — a suite that recognizes a file +> owns its format (enforced end-to-end by `tests/test_plugin_protocol.py`). +> Returning `None` for files that are not yours is part of the contract. --- @@ -231,16 +275,23 @@ class MyMetric(BaseMetric): def name(self) -> str: return "my_metric" - def compute(self, predictions: list[str], references: list[str]) -> float: - import json + def compute(self, samples) -> float: + # A sample is whatever record your inference step produces — + # dicts recommended; put references inside the record. + if not samples: + return 0.0 correct = sum( - json.loads(p) == json.loads(r) - for p, r in zip(predictions, references) + 1 + for s in samples + if isinstance(s, dict) and s.get("prediction") == s.get("reference") ) - return correct / len(predictions) if predictions else 0.0 + return correct / len(samples) ``` -`compute()` receives predictions and references as JSON-serialized strings. +`compute()` receives the task's per-sample records directly (BaseMetric API +v2 — see `oellm.core.CORE_API_VERSION`). Multi-reference tasks put their +references inside each record; handle invalid records (`None`, parse +failures) deliberately and document the choice. --- diff --git a/oellm/contrib/regiondial_bench/metrics.py b/oellm/contrib/regiondial_bench/metrics.py index 9ae0dde8..625ff9ed 100644 --- a/oellm/contrib/regiondial_bench/metrics.py +++ b/oellm/contrib/regiondial_bench/metrics.py @@ -4,21 +4,20 @@ per-sample inference results. These are used by ``suite._aggregate_shards()`` to score any model's predictions on RegionDial-Bench. -Input format ------------- -Each sample is a JSON-serialised dict containing pre-computed fields from the -inference script:: +Input format (BaseMetric API v2) +-------------------------------- +Each sample is a dict of pre-computed fields from the inference script:: { "intersection": 12345, "union": 23456, "bbox_iou": 0.73, - "round": 1 + "image_id": "0001", } -``predictions`` passed to ``compute()`` are ``list[str]`` — one JSON string -per sample. ``references`` are unused (empty strings) since ground truth is -already folded into the intersection/union computation by the inference script. +Entries that are not dicts (``None``, upstream parse failures) are treated +as failed samples: the per-sample metrics (gIoU, bbox_AP, pass rates) score +them 0, while cIoU — a corpus-level ratio — excludes them from both sums. Metrics ------- @@ -30,22 +29,15 @@ from __future__ import annotations -import json +from collections.abc import Sequence +from typing import Any from oellm.core.base_metric import BaseMetric -def _parse_sample(s: str) -> dict | None: - """Parse a JSON-serialised sample dict. Returns None on failure.""" - if not s or s.strip() in ("null", "none", ""): - return None - try: - val = json.loads(s) - if isinstance(val, dict): - return val - except (json.JSONDecodeError, ValueError, TypeError): - pass - return None +def _as_sample(s: Any) -> dict | None: + """Return *s* if it is a usable sample record, else ``None``.""" + return s if isinstance(s, dict) else None def _mask_iou(sample: dict) -> float: @@ -60,19 +52,20 @@ def _mask_iou(sample: dict) -> float: class GIoU(BaseMetric): """Mean per-sample mask IoU (gIoU as reported in RegionDial-Bench). - Formula: ``mean(intersection_i / union_i)`` over all samples. + Formula: ``mean(intersection_i / union_i)`` over all samples; failed + samples count as 0. """ @property def name(self) -> str: return "gIoU" - def compute(self, predictions: list[str], references: list[str]) -> float: - if not predictions: + def compute(self, samples: Sequence[Any]) -> float: + if not samples: return 0.0 ious = [] - for s in predictions: - sample = _parse_sample(s) + for s in samples: + sample = _as_sample(s) ious.append(_mask_iou(sample) if sample else 0.0) return sum(ious) / len(ious) @@ -80,18 +73,19 @@ def compute(self, predictions: list[str], references: list[str]) -> float: class CIoU(BaseMetric): """cIoU as reported in RegionDial-Bench. - Formula: ``sum(all intersections) / sum(all unions)``. + Formula: ``sum(all intersections) / sum(all unions)``; failed samples + are excluded from both sums (they carry no area information). """ @property def name(self) -> str: return "cIoU" - def compute(self, predictions: list[str], references: list[str]) -> float: + def compute(self, samples: Sequence[Any]) -> float: total_intersection = 0.0 total_union = 0.0 - for s in predictions: - sample = _parse_sample(s) + for s in samples: + sample = _as_sample(s) if sample is None: continue total_intersection += sample.get("intersection", 0) @@ -105,22 +99,22 @@ class BboxAP(BaseMetric): """Binarised bounding-box AP at IoU threshold 0.5. Uses the pre-computed ``bbox_iou`` field from the inference script. - Formula: ``mean(bbox_iou_i > 0.5)``. + Formula: ``mean(bbox_iou_i > 0.5)``; failed samples count as misses. """ @property def name(self) -> str: return "bbox_AP" - def compute(self, predictions: list[str], references: list[str]) -> float: - if not predictions: + def compute(self, samples: Sequence[Any]) -> float: + if not samples: return 0.0 hits = 0 - for s in predictions: - sample = _parse_sample(s) + for s in samples: + sample = _as_sample(s) if sample and sample.get("bbox_iou", 0) > 0.5: hits += 1 - return hits / len(predictions) + return hits / len(samples) class PassRate(BaseMetric): @@ -141,12 +135,12 @@ def name(self) -> str: label = f"{t:.10g}" return f"pass_rate_{label}" - def compute(self, predictions: list[str], references: list[str]) -> float: - if not predictions: + def compute(self, samples: Sequence[Any]) -> float: + if not samples: return 0.0 hits = 0 - for s in predictions: - sample = _parse_sample(s) + for s in samples: + sample = _as_sample(s) if sample and _mask_iou(sample) > self._threshold: hits += 1 - return hits / len(predictions) + return hits / len(samples) diff --git a/oellm/contrib/regiondial_bench/suite.py b/oellm/contrib/regiondial_bench/suite.py index 3197c5b9..67e03bb1 100644 --- a/oellm/contrib/regiondial_bench/suite.py +++ b/oellm/contrib/regiondial_bench/suite.py @@ -381,9 +381,6 @@ def _aggregate_shards( "Aggregating %d samples from %d shards", len(all_samples), len(shard_files) ) - samples = [json.dumps(s) for s in all_samples] - empty_refs = [""] * len(samples) - aggregate_metrics = [ GIoU(), CIoU(), @@ -396,7 +393,7 @@ def _aggregate_shards( metrics: dict[str, float] = {} for m in aggregate_metrics: - val = m.compute(samples, empty_refs) + val = m.compute(all_samples) metrics[m.name] = val logger.debug("%s = %.4f", m.name, val) @@ -404,20 +401,19 @@ def _aggregate_shards( # the output (mirrors calculate_iou_with_bbox_by_turns.py). The inference # script emits turns in sequential order per image, so the k-th time an # image_id appears corresponds to turn k (1-indexed). - rounds_map: dict[int, list[str]] = defaultdict(list) + rounds_map: dict[int, list[dict]] = defaultdict(list) image_turn_counter: dict[str, int] = {} - for sample_dict, sample_str in zip(all_samples, samples, strict=True): + for sample_dict in all_samples: image_id = str(sample_dict.get("image_id", "")) image_turn_counter[image_id] = image_turn_counter.get(image_id, 0) + 1 rnd = image_turn_counter[image_id] - rounds_map[rnd].append(sample_str) + rounds_map[rnd].append(sample_dict) per_round_metrics = [GIoU(), BboxAP()] for rnd in sorted(rounds_map): rnd_samples = rounds_map[rnd] - rnd_refs = [""] * len(rnd_samples) for m in per_round_metrics: - val = m.compute(rnd_samples, rnd_refs) + val = m.compute(rnd_samples) metrics[f"{m.name}_R{rnd}"] = val logger.debug("%s_R%d = %.4f", m.name, rnd, val) diff --git a/oellm/core/__init__.py b/oellm/core/__init__.py index 08a238dd..8d3e57cb 100644 --- a/oellm/core/__init__.py +++ b/oellm/core/__init__.py @@ -1,5 +1,23 @@ +"""Core plugin interfaces. + +``CORE_API_VERSION`` marks the plugin-API freeze (plan item G4): breaking +changes to :class:`BaseTask` / :class:`BaseMetric` / :class:`BaseModelAdapter` +after 1.0 require a deprecation cycle. The v2 ``BaseMetric`` contract +(per-sample records instead of parallel prediction/reference string lists) +landed *before* this freeze, precisely so external contributions never build +against the narrower signature. +""" + from oellm.core.base_metric import BaseMetric -from oellm.core.base_model_adapter import BaseModelAdapter +from oellm.core.base_model_adapter import BaseModelAdapter, DefaultHFAdapter from oellm.core.base_task import BaseTask -__all__ = ["BaseTask", "BaseMetric", "BaseModelAdapter"] +CORE_API_VERSION = "1.0" + +__all__ = [ + "CORE_API_VERSION", + "BaseMetric", + "BaseModelAdapter", + "BaseTask", + "DefaultHFAdapter", +] diff --git a/oellm/core/base_metric.py b/oellm/core/base_metric.py index 8a5b5568..a1bc6cd4 100644 --- a/oellm/core/base_metric.py +++ b/oellm/core/base_metric.py @@ -1,4 +1,6 @@ from abc import ABC, abstractmethod +from collections.abc import Sequence +from typing import Any class BaseMetric(ABC): @@ -8,9 +10,21 @@ class BaseMetric(ABC): supported by lm-eval or lmms-eval (e.g. a custom IoU score for grounding benchmarks, or a domain-specific accuracy metric). - The ``compute`` method must return a scalar in [0, 1] by convention, - though higher-range metrics (e.g. OCRBench score /1000) are allowed when - the metric name makes the range unambiguous. + Contract (API v2 — see ``oellm.core.CORE_API_VERSION``): ``compute`` + receives the task's per-sample records and returns one scalar. A *sample* + is whatever record the task's inference step produces — a dict is + recommended. This replaces the old ``(predictions: list[str], + references: list[str])`` signature, which could not express + multi-reference tasks (VQA accuracy, ANLS), weighted aggregation, or + non-mean corpus aggregates (e.g. summed-IoU ratios), and forced + implementations to smuggle structured data through JSON strings. + + Conventions: + - Return a scalar in [0, 1] unless the metric name makes another range + unambiguous (e.g. an explicit ``/1000`` score). + - Entries that are not valid sample records (``None``, parse failures) + must be handled deliberately — scored as failures or excluded — and + the choice documented on the metric class. Example:: @@ -19,15 +33,16 @@ class ExactMatchMetric(BaseMetric): def name(self) -> str: return "exact_match" - def compute( - self, - predictions: list[str], - references: list[str], - ) -> float: - if not predictions: + def compute(self, samples: Sequence[Any]) -> float: + if not samples: return 0.0 - correct = sum(p == r for p, r in zip(predictions, references)) - return correct / len(predictions) + correct = sum( + 1 + for s in samples + if isinstance(s, dict) + and s.get("prediction") == s.get("reference") + ) + return correct / len(samples) """ @property @@ -36,17 +51,14 @@ def name(self) -> str: """Unique metric identifier, e.g. ``"vqa_score"`` or ``"anls"``.""" @abstractmethod - def compute( - self, - predictions: list[str], - references: list[str], - ) -> float: - """Compute the metric score. + def compute(self, samples: Sequence[Any]) -> float: + """Compute the metric over per-sample records. Args: - predictions: Model-generated answers (one per sample). - references: Ground-truth answers (one per sample). + samples: One record per evaluated sample (dicts recommended; + multi-reference tasks put their references inside the record). Returns: - Scalar score. Conventionally in [0, 1]; higher is better. + Scalar score. Conventionally in [0, 1]; higher is better unless + the metric name says otherwise (e.g. WER). """ diff --git a/oellm/core/base_model_adapter.py b/oellm/core/base_model_adapter.py index fb0615f9..24896364 100644 --- a/oellm/core/base_model_adapter.py +++ b/oellm/core/base_model_adapter.py @@ -6,7 +6,10 @@ class BaseModelAdapter(ABC): """Abstract base class for model adapters. Translates a model path/config into engine-specific argument strings - passed on the command line. + passed on the command line. The built-in execution path consumes these + through :class:`DefaultHFAdapter` (rendered into ``template.sbatch`` by + the scheduler); contrib suites consume :meth:`to_contrib_flags` via their + ``detect_model_flags()``. Example:: @@ -45,6 +48,13 @@ def to_lmms_eval_args(self) -> str: Example: ``"pretrained=/path/to/model"`` """ + def to_evalchemy_args(self) -> str: + """Return the ``--model_args`` string for the evalchemy engine. + + Defaults to the lm-eval string (evalchemy is a forked lm-eval). + """ + return self.to_lm_eval_args() + def to_contrib_flags(self) -> str | None: """Return the model-type flag for contrib suite routing. @@ -55,3 +65,41 @@ def to_contrib_flags(self) -> str | None: Returns ``None`` by default (no model-type distinction needed). """ return None + + +class DefaultHFAdapter(BaseModelAdapter): + """Built-in HuggingFace adapter — the single source of the engine + ``--model_args`` strings rendered into ``template.sbatch``. + + The scheduler instantiates it once per run with the literal bash + placeholder ``$model_path`` (each CSV row substitutes its model at + runtime on the compute node); ``extra_args`` carries run-level additions + such as quantization (``",load_in_4bit=True"``). Per-model argument + differentiation arrives with the plugin-protocol ``eval_args`` channel. + """ + + def __init__( + self, + model_path: str | Path = "$model_path", + *, + trust_remote_code: bool = True, + extra_args: str = "", + ) -> None: + self._path = model_path + self._trust = trust_remote_code + self._extra = extra_args + + @property + def model_path(self) -> str | Path: + return self._path + + def to_lm_eval_args(self) -> str: + return f'pretrained="{self._path}",trust_remote_code={self._trust}{self._extra}' + + def to_lmms_eval_args(self) -> str: + # $_lmms_extra_args is the bash-side per-family hook filled in + # template.sbatch (llava model_name workaround, qwen frame cap). + return f"pretrained={self._path},device_map=auto$_lmms_extra_args{self._extra}" + + def to_evalchemy_args(self) -> str: + return f"trust_remote_code={self._trust},pretrained={self._path}{self._extra}" diff --git a/oellm/core/base_task.py b/oellm/core/base_task.py index 2d779e44..4f433f26 100644 --- a/oellm/core/base_task.py +++ b/oellm/core/base_task.py @@ -138,7 +138,11 @@ def to_task_groups_dict(cls) -> dict: """ inst = cls() - task_entry: dict = {"task": inst.name} + # The engine-facing name goes into the task entry: it is what lands + # in jobs.csv, what the engine CLI receives, and what results come + # back under — so collect and --check stay consistent. Overriding + # engine_task_name is the plan's "one-liner plugin" pathway. + task_entry: dict = {"task": inst.engine_task_name} if inst.dataset_specs: task_entry["dataset"] = inst.dataset_specs[0].repo_id if inst.dataset_specs[0].subset: @@ -158,5 +162,5 @@ def to_task_groups_dict(cls) -> dict: result: dict = {"task_groups": {inst.task_group_name: task_group}} if inst.primary_metric: - result["task_metrics"] = {inst.name: inst.primary_metric} + result["task_metrics"] = {inst.engine_task_name: inst.primary_metric} return result diff --git a/oellm/main.py b/oellm/main.py index dacc1cea..9e1f1c43 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -89,7 +89,10 @@ def schedule_evals( skip_checks: If True, skip container image, environment pre-flight (engine availability in the venv/container per scheduled suite), model validation, and dataset pre-download checks for faster execution. - trust_remote_code: If True, trust remote code when downloading datasets. Default is True. Workflow might fail if set to False. + trust_remote_code: If True, trust remote code when downloading datasets AND + at eval time for lm_eval / lighteval / evalchemy (previously hardcoded + True at eval time). lmms-eval adapters manage their own loading. + Default True. Workflow might fail if set to False. venv_path: Path to a Python virtual environment. If provided, evaluations run directly using this venv instead of inside a Singularity/Apptainer container. lm_eval_include_path: Path to a directory containing custom lm_eval task YAML definitions. diff --git a/oellm/registry.py b/oellm/registry.py index bbbba335..91d84a38 100644 --- a/oellm/registry.py +++ b/oellm/registry.py @@ -24,6 +24,8 @@ ``parse_results(data: dict) -> tuple | None`` Try to parse a raw JSON dict produced by this suite. Returns ``(model_id, task_name, n_shot, {metric: value})`` or ``None``. + Called by :func:`oellm.results.collect_results` as a first-chance + parser — a suite that claims a file owns its format outright. Optional ~~~~~~~~ @@ -35,6 +37,12 @@ Return a model-type suffix for the ``eval_suite`` column (e.g. ``"vision_reasoner"``), or ``None``. +``LMMS_MODEL_ADAPTERS: list[tuple[list[str], str]]`` + lmms-eval adapter detection patterns (same shape as + ``oellm.constants.LMMS_MODEL_ADAPTERS``). Consulted BEFORE the built-in + table by ``detect_lmms_model_type`` so a plugin can route new model + families without core edits. + Adding a new benchmark ---------------------- Drop files into ``oellm/contrib//``. No core file changes required. @@ -117,6 +125,19 @@ def get_all_suites() -> list[types.ModuleType]: return list(_discover().values()) +def get_lmms_adapter_overrides() -> list[tuple[list[str], str]]: + """Contrib-registered lmms-eval adapter detection patterns. + + Aggregates the optional ``LMMS_MODEL_ADAPTERS`` export of every + discovered suite; entries take precedence over the built-in table in + ``oellm.constants``. + """ + overrides: list[tuple[list[str], str]] = [] + for mod in _discover().values(): + overrides.extend(getattr(mod, "LMMS_MODEL_ADAPTERS", None) or []) + return overrides + + def get_all_task_groups() -> dict: """Merge TASK_GROUPS from all discovered suites into a single dict. diff --git a/oellm/resources/custom_lm_eval_tasks/tabfact/tabfact.yaml b/oellm/resources/custom_lm_eval_tasks/tabfact/tabfact.yaml new file mode 100644 index 00000000..d75f58c4 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/tabfact/tabfact.yaml @@ -0,0 +1,24 @@ +# TabFact — table fact verification (wenhu/tab_fact, CC-BY-4.0, ungated). +# Reference custom lm-eval task: is a statement entailed or refuted +# by a Wikipedia table? MCQ loglikelihood over the two verdicts; few-shot +# demonstrations come from the TRAIN split (clean split hygiene). Label +# mapping: 0 = refuted, 1 = entailed — matches doc_to_choice order. +task: tabfact +dataset_path: wenhu/tab_fact +dataset_name: tab_fact +dataset_kwargs: + trust_remote_code: true +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +fewshot_split: train +doc_to_text: !function utils.doc_to_text +doc_to_choice: ["refuted", "entailed"] +doc_to_target: label +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/oellm/resources/custom_lm_eval_tasks/tabfact/utils.py b/oellm/resources/custom_lm_eval_tasks/tabfact/utils.py new file mode 100644 index 00000000..9f471061 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/tabfact/utils.py @@ -0,0 +1,35 @@ +"""TabFact prompt construction. + +``wenhu/tab_fact`` stores tables in ``table_text``: rows separated by +newlines, cells by ``#``, first row is the header. Serialized here as a +pipe table capped at ``_MAX_ROWS`` rows so pathological tables cannot blow +the context window (the cap is stated in the prompt when it triggers). +""" + +_MAX_ROWS = 30 + + +def _serialize_table(table_text: str) -> str: + rows = [r for r in str(table_text).split("\n") if r.strip()] + lines: list[str] = [] + for i, row in enumerate(rows): + cells = [c.strip() for c in row.split("#")] + lines.append(" | ".join(cells)) + if i == 0 and len(rows) > 1: + lines.append("-" * min(80, max(3, len(lines[0])))) + if i + 1 >= _MAX_ROWS and len(rows) > _MAX_ROWS: + lines.append(f"... ({len(rows) - _MAX_ROWS} more rows)") + break + return "\n".join(lines) + + +def doc_to_text(doc) -> str: + caption = str(doc.get("table_caption", "") or "").strip() + table = _serialize_table(doc.get("table_text", "")) + head = f"Table caption: {caption}\n" if caption else "" + return ( + f"{head}Table:\n{table}\n" + f"Statement: {doc['statement']}\n" + "Question: Is the statement entailed or refuted by the table?\n" + "Answer:" + ) diff --git a/oellm/resources/custom_lm_eval_tasks/timeseriesexam/timeseriesexam.yaml b/oellm/resources/custom_lm_eval_tasks/timeseriesexam/timeseriesexam.yaml new file mode 100644 index 00000000..ccea8c2a --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/timeseriesexam/timeseriesexam.yaml @@ -0,0 +1,21 @@ +# TimeSeriesExam (AutonLab/TimeSeriesExam1, arXiv 2410.14752) — MCQ time +# series *understanding* for LLMs: trend, seasonality, anomalies, causality. +# Evaluates existing text models on time-series understanding; benchmarks +# for dedicated forecasting models (fev/GIFT-Eval) are a separate, future +# engine decision. +# Dataset is parquet-native and ungated; test split only → strictly 0-shot. +# Series are uniformly subsampled to 128 points (see utils.py — fixed policy, +# identical for all models). +task: timeseriesexam +dataset_path: AutonLab/TimeSeriesExam1 +test_split: test +output_type: multiple_choice +doc_to_text: !function utils.doc_to_text +doc_to_choice: !function utils.doc_to_choice +doc_to_target: !function utils.doc_to_target +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/oellm/resources/custom_lm_eval_tasks/timeseriesexam/utils.py b/oellm/resources/custom_lm_eval_tasks/timeseriesexam/utils.py new file mode 100644 index 00000000..9098ba38 --- /dev/null +++ b/oellm/resources/custom_lm_eval_tasks/timeseriesexam/utils.py @@ -0,0 +1,64 @@ +"""TimeSeriesExam prompt construction (AutonLab/TimeSeriesExam1). + +Rows carry either one series (``ts``) or a pair (``ts1``/``ts2``), each +1000–2500 raw floats — far too long to inline. Serialization policy +(fixed and versioned; identical for every model, so scores stay +comparable): uniform subsample to at most ``_MAX_POINTS`` values in +original order, formatted to 4 significant digits, with the original +length stated in the prompt. + +MCQ is MMLU-style: options are printed lettered, the model scores +single-letter continuations — immune to option-length bias, and robust +to the dataset's variable option counts (binary and 4-way questions). +""" + +_MAX_POINTS = 128 + + +def _fmt_series(vals) -> str: + vals = list(vals or []) + n = len(vals) + if n > _MAX_POINTS: + step = n / _MAX_POINTS + vals = [vals[int(i * step)] for i in range(_MAX_POINTS)] + return ", ".join(f"{v:.4g}" for v in vals) + + +def _series_block(doc) -> str: + parts = [] + if doc.get("ts"): + parts.append( + f"Time series ({len(doc['ts'])} points, uniformly subsampled to " + f"{min(len(doc['ts']), _MAX_POINTS)}):\n{_fmt_series(doc['ts'])}" + ) + if doc.get("ts1"): + parts.append( + f"Series A ({len(doc['ts1'])} points, uniformly subsampled to " + f"{min(len(doc['ts1']), _MAX_POINTS)}):\n{_fmt_series(doc['ts1'])}" + ) + if doc.get("ts2"): + parts.append( + f"Series B ({len(doc['ts2'])} points, uniformly subsampled to " + f"{min(len(doc['ts2']), _MAX_POINTS)}):\n{_fmt_series(doc['ts2'])}" + ) + return "\n\n".join(parts) + + +def doc_to_text(doc) -> str: + options = "\n".join(f"{chr(65 + i)}. {opt}" for i, opt in enumerate(doc["options"])) + return ( + f"{_series_block(doc)}\n\n" + f"Question: {doc['question']}\n" + f"Options:\n{options}\n" + "Answer:" + ) + + +def doc_to_choice(doc) -> list[str]: + return [f" {chr(65 + i)}" for i in range(len(doc["options"]))] + + +def doc_to_target(doc) -> int: + """Index of the gold option; the answer field stores the option text.""" + options = [str(o).strip() for o in doc["options"]] + return options.index(str(doc["answer"]).strip()) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 96a99086..0f1a7624 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -528,6 +528,11 @@ task_metrics: mgsm_native_cot_es: exact_match,flexible-extract mgsm_native_cot_fr: exact_match,flexible-extract + # Custom lm-eval task — definition in custom_lm_eval_tasks/tabfact/. + tabfact: acc + # Custom lm-eval task — definition in custom_lm_eval_tasks/timeseriesexam/. + timeseriesexam: acc + task_groups: sib200-eu: description: "SIB-200 European language topic classification tasks (0-shot)" @@ -1460,6 +1465,22 @@ task_groups: tasks: - task: "polymath_{lang}_top" subset: "{lang}" + tabular-tabfact: + description: "TabFact: is a statement entailed or refuted by a Wikipedia table. Few-shot drawn from the train split." + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: tabfact + dataset: wenhu/tab_fact + subset: tab_fact + + timeseries-tsexam: + description: "TimeSeriesExam (AutonLab, arXiv 2410.14752): MCQ time-series understanding for LLMs — trend, seasonality, anomalies, causality. Strictly 0-shot (test split only); series subsampled to 128 points." + suite: lm-eval-harness + n_shots: [0] + tasks: + - task: timeseriesexam + dataset: AutonLab/TimeSeriesExam1 super_groups: oellm-multilingual: diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index aff0f358..fc738581 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -22,6 +22,8 @@ export LIMIT="{limit}" # Quantization request ("4bit" / "8bit" / empty). HF-style engines receive it # via --model_args below; contrib plugins may opt in by reading this variable. export OELLM_QUANTIZATION="{quantization}" +# Non-empty => pass --trust_remote_code to lm_eval (dataset-side trust). +LM_EVAL_TRC="{lm_eval_trc}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" @@ -172,11 +174,11 @@ do echo "----------------------------------------------------" echo "lm_eval Execution" run_python -m lm_eval --model hf \ - --model_args pretrained="$model_path",trust_remote_code=True{quantization_model_args} \ + --model_args {lm_eval_model_args} \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "{evals_dir}/$(openssl rand -hex 5).json" \ - --trust_remote_code \ + ${{LM_EVAL_TRC:+--trust_remote_code}} \ --batch_size auto \ ${{LM_EVAL_INCLUDE_PATH:+--include_path $LM_EVAL_INCLUDE_PATH}} \ ${{LIMIT:+--limit $LIMIT}} @@ -207,7 +209,7 @@ do if [ -n "$VENV_PATH" ]; then source "$VENV_PATH/bin/activate" _maybe_timeout lighteval accelerate \ - "model_name=$model_path,trust_remote_code=True,{additional_model_args}" \ + "model_name=$model_path,{lighteval_trc}{additional_model_args}" \ "$LIGHT_TASK_ARG" \ --load-tasks-multilingual \ --output-dir "$RESULTS_SUBDIR" \ @@ -220,7 +222,7 @@ do $EVAL_SIF_PATH \ env CUDA_VISIBLE_DEVICES=$GPU_DEVICES \ lighteval accelerate \ - "model_name=$model_path,{additional_model_args}" \ + "model_name=$model_path,{lighteval_trc}{additional_model_args}" \ "$LIGHT_TASK_ARG" \ --load-tasks-multilingual \ --output-dir "$RESULTS_SUBDIR" \ @@ -250,7 +252,7 @@ do run_python -m lmms_eval \ --model "$_lmms_adapter" \ - --model_args "pretrained=$model_path,device_map=auto$_lmms_extra_args{quantization_model_args}" \ + --model_args "{lmms_eval_model_args}" \ --tasks "$task_path" \ --num_fewshot "$n_shot" \ --output_path "$OUTPUT_JSON" \ @@ -269,7 +271,7 @@ do $MULTI_GPU_FLAG -m eval.eval \ --model hf \ --tasks "$task_path" \ - --model_args "trust_remote_code=True,pretrained=$model_path{quantization_model_args}" \ + --model_args "{evalchemy_model_args}" \ --batch_size auto \ --output_path "$RESULTS_SUBDIR" \ ${{LIMIT:+--limit $LIMIT}} < /dev/null diff --git a/oellm/results.py b/oellm/results.py index 53f5ce84..fc6ceda5 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -264,6 +264,31 @@ def _load_task_metrics() -> dict: return task_metrics +def _try_contrib_parse(data: dict) -> tuple[str, str, int, dict] | None: + """First-chance parse via contrib suites' ``parse_results()``. + + Lets a plugin own its output format outright instead of relying on the + generic lmms-shaped heuristics in :func:`collect_results`. A broken + parser must never break collect: exceptions are logged and skipped. + """ + from oellm.registry import get_all_suites + + for mod in get_all_suites(): + parser = getattr(mod, "parse_results", None) + if parser is None: + continue + try: + parsed = parser(data) + except Exception as e: # noqa: BLE001 — plugin code; isolate failures + logging.debug( + f"parse_results of {getattr(mod, 'SUITE_NAME', mod)!r} raised: {e}" + ) + continue + if parsed: + return parsed + return None + + def collect_results( results_dir: str, output_csv: str = "eval_results.csv", @@ -380,6 +405,34 @@ def collect_results( ) continue + # First-chance: a contrib suite may claim this file outright via its + # parse_results() protocol member and own the format end-to-end. + _contrib_parsed = _try_contrib_parse(data) + if _contrib_parsed is not None: + _c_model, _c_task, _c_n_shot, _c_metrics = _contrib_parsed + performance, metric_name = _resolve_metric(_c_task, _c_metrics, task_metrics) + if performance is not None: + if check: + completed_jobs.add((_c_model, _c_task, _c_n_shot)) + rows.append( + { + "model_name": _c_model, + "task": _c_task, + "n_shot": _c_n_shot, + "performance": performance, + "performance_normalized": _normalize_to_100( + performance, metric_name, _c_task + ), + "metric_name": metric_name if metric_name is not None else "", + } + ) + else: + logging.warning( + f"No numeric metric for contrib-parsed '{_c_task}' in " + f"{json_file.name} — check the suite's task_metrics entry" + ) + continue + # Model name lives in different keys depending on the harness: # - lmms-eval: model_name_or_path is the checkpoint, model_name is the # adapter class (e.g. "llava_hf") diff --git a/oellm/scheduler.py b/oellm/scheduler.py index 4160af2d..3d1544f0 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -1,3 +1,4 @@ +import getpass import json import logging import math @@ -14,6 +15,7 @@ from oellm import __version__ from oellm.constants import EvaluationJob +from oellm.core import DefaultHFAdapter from oellm.results import _collector_git_commit, _load_task_metrics from oellm.runner import EvalRunner from oellm.task_groups import ( @@ -178,7 +180,11 @@ def schedule_evals( download_only: If True, only download the datasets and models and exit. dry_run: If True, generate the SLURM script but don't submit it to the scheduler. skip_checks: If True, skip container image, model validation, and dataset pre-download checks for faster execution. - trust_remote_code: If True, trust remote code when downloading datasets. Default is True. Workflow might fail if set to False. + trust_remote_code: If True, trust remote code when downloading datasets AND + at eval time for lm_eval / lighteval / evalchemy (threaded through the + adapter-rendered model args; previously hardcoded True at eval time). + lmms-eval adapters manage their own loading. Default True. Workflow + might fail if set to False. venv_path: Path to a Python virtual environment. If provided, evaluations run directly using this venv instead of inside a Singularity/Apptainer container. lm_eval_include_path: Path to a directory containing custom lm_eval task YAML definitions. @@ -380,6 +386,20 @@ def schedule_evals( f"OELLM_QUANTIZATION env var exported to the job)." ) + # Engine --model_args now come from the adapter layer (single source of + # truth — template.sbatch receives only rendered strings; previously the + # strings were hardcoded in bash and BaseModelAdapter was dead code). + # "$model_path" stays a literal bash placeholder substituted per CSV row + # on the compute node. trust_remote_code now genuinely governs eval time + # for lm_eval / lighteval / evalchemy — it used to be hardcoded True. + _adapter = DefaultHFAdapter( + trust_remote_code=trust_remote_code, extra_args=quantization_model_args + ) + lm_eval_model_args = _adapter.to_lm_eval_args() + lmms_eval_model_args = _adapter.to_lmms_eval_args() + evalchemy_model_args = _adapter.to_evalchemy_args() + lighteval_trc = "trust_remote_code=True," if trust_remote_code else "" + if not skip_checks: # Verify the runtime can actually execute the scheduled suites before # any network work: missing engines otherwise fail row-by-row on the @@ -576,6 +596,9 @@ def _lower_suite_only(s: str) -> str: "schema": 1, "created_at": timestamp, "hostname": socket.gethostname(), + "submitted_by": getpass.getuser() + if not os.environ.get("OELLM_SUBMITTED_BY") + else os.environ["OELLM_SUBMITTED_BY"], "oellm_version": __version__, "scheduler_git_commit": _collector_git_commit(), "eval_suites": sorted({str(j.eval_suite) for j in expanded_eval_jobs}), @@ -590,6 +613,12 @@ def _lower_suite_only(s: str) -> str: "hf_hub_offline": _resolve_hf_hub_offline(local), "quantization": quantization or None, "row_timeout": os.environ.get("ROW_TIMEOUT"), + "trust_remote_code": trust_remote_code, + "engine_model_args": { + "lm_eval": lm_eval_model_args, + "lmms_eval": lmms_eval_model_args, + "evalchemy": evalchemy_model_args, + }, "model_revisions": model_revisions, "engine_versions": {} if skip_checks else _probe_engine_versions(venv_path), "container_image": None if venv_path else os.environ.get("EVAL_CONTAINER_IMAGE"), @@ -615,7 +644,11 @@ def _lower_suite_only(s: str) -> str: additional_model_args=additional_model_args, evalchemy_dir=os.environ.get("EVALCHEMY_DIR", "/opt/evalchemy"), quantization=quantization, - quantization_model_args=quantization_model_args, + lm_eval_model_args=lm_eval_model_args, + lmms_eval_model_args=lmms_eval_model_args, + evalchemy_model_args=evalchemy_model_args, + lighteval_trc=lighteval_trc, + lm_eval_trc="1" if trust_remote_code else "", ) # Drop optional #SBATCH directives whose env var is unset, so safe_substitute diff --git a/tests/test_base_interfaces.py b/tests/test_base_interfaces.py index 480d930d..3bc705f1 100644 --- a/tests/test_base_interfaces.py +++ b/tests/test_base_interfaces.py @@ -37,12 +37,10 @@ class ExactMatchMetric(BaseMetric): def name(self) -> str: return "exact_match" - def compute(self, predictions: list[str], references: list[str]) -> float: - if not predictions: + def compute(self, samples) -> float: + if not samples: return 0.0 - return sum(p == r for p, r in zip(predictions, references, strict=True)) / len( - predictions - ) + return sum(1 for s in samples if s["prediction"] == s["reference"]) / len(samples) class HFAdapter(BaseModelAdapter): @@ -145,23 +143,38 @@ def test_concrete_subclass_instantiates(self): def test_compute_perfect_score(self): m = ExactMatchMetric() - assert m.compute(["a", "b", "c"], ["a", "b", "c"]) == 1.0 + assert ( + m.compute([{"prediction": x, "reference": x} for x in ("a", "b", "c")]) == 1.0 + ) def test_compute_zero_score(self): m = ExactMatchMetric() - assert m.compute(["a", "b"], ["x", "y"]) == 0.0 + assert ( + m.compute( + [ + {"prediction": "a", "reference": "x"}, + {"prediction": "b", "reference": "y"}, + ] + ) + == 0.0 + ) def test_compute_partial_score(self): m = ExactMatchMetric() - assert m.compute(["a", "b"], ["a", "x"]) == pytest.approx(0.5) + assert m.compute( + [ + {"prediction": "a", "reference": "a"}, + {"prediction": "b", "reference": "x"}, + ] + ) == pytest.approx(0.5) def test_compute_empty_returns_zero(self): m = ExactMatchMetric() - assert m.compute([], []) == 0.0 + assert m.compute([]) == 0.0 def test_missing_abstract_name_raises(self): class BadMetric(BaseMetric): - def compute(self, predictions, references): + def compute(self, samples): return 0.0 with pytest.raises(TypeError): diff --git a/tests/test_plugin_protocol.py b/tests/test_plugin_protocol.py new file mode 100644 index 00000000..0e95f686 --- /dev/null +++ b/tests/test_plugin_protocol.py @@ -0,0 +1,258 @@ +"""End-to-end conformance test for the contrib plugin protocol. + +A synthetic third-party plugin is materialized on disk and discovered through +the real registry; every documented protocol member is then exercised on its +real consumer path: TASK_GROUPS via BaseTask (including the engine_task_name +override — the plan's "one-liner plugin" pathway), LMMS_MODEL_ADAPTERS +overrides via detect_lmms_model_type, detect_model_flags via BaseModelAdapter +in EvalRunner.resolve_suite, BaseMetric (API v2) inside run(), and +parse_results as collect_results' first-chance parser. If any protocol member +loses its consumer again (the dead-extension-point failure class), this +file fails. +""" + +import csv +import importlib.util +import json +import sys + +import pytest + + +def _load_task_utils(task_dir: str): + """Import a custom task's utils.py under a unique module name — a bare + ``import utils`` would collide across task dirs in sys.modules.""" + path = f"oellm/resources/custom_lm_eval_tasks/{task_dir}/utils.py" + spec = importlib.util.spec_from_file_location(f"{task_dir}_task_utils", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +SUITE_SRC = """ +from pathlib import Path + +from oellm.core import BaseMetric, BaseModelAdapter, BaseTask + +SUITE_NAME = "toy_suite" +CLUSTER_ENV_VARS: list[str] = [] +LMMS_MODEL_ADAPTERS = [(["toy-vlm"], "toy_adapter")] + + +class ToyTask(BaseTask): + @property + def name(self): + return "toy_task" + + @property + def engine_task_name(self): + return "toy_task_engine" + + @property + def suite(self): + return SUITE_NAME + + @property + def n_shots(self): + return [0] + + @property + def primary_metric(self): + return "toy_score" + + +class ToyAdapter(BaseModelAdapter): + def __init__(self, path): + self._p = path + + @property + def model_path(self): + return self._p + + def to_lm_eval_args(self): + return f"pretrained={self._p}" + + def to_lmms_eval_args(self): + return f"pretrained={self._p}" + + def to_contrib_flags(self): + return "toy_backend" if "toy" in str(self._p).lower() else None + + +class ToyScore(BaseMetric): + @property + def name(self): + return "toy_score" + + def compute(self, samples): + if not samples: + return 0.0 + return sum(s["ok"] for s in samples) / len(samples) + + +TASK_GROUPS = ToyTask.to_task_groups_dict() + + +def detect_model_flags(model_path): + return ToyAdapter(model_path).to_contrib_flags() + + +def run(*, model_path, task, n_shot, output_path, model_flags, env): + import json as _json + + score = ToyScore().compute([{"ok": 1}, {"ok": 0}]) + Path(output_path).write_text( + _json.dumps( + { + "model_name_or_path": str(model_path), + "results": {task: {"toy_score": score, "backend": model_flags}}, + "configs": {task: {"num_fewshot": n_shot}}, + } + ) + ) + + +def parse_results(data): + results = data.get("results", {}) + if not isinstance(results, dict): + return None + for tname, tres in results.items(): + if isinstance(tname, str) and tname.startswith("toy_task") and "toy_score" in tres: + n_shot = data.get("configs", {}).get(tname, {}).get("num_fewshot", 0) + return (data.get("model_name_or_path", "?"), tname, int(n_shot), dict(tres)) + return None +""" + + +@pytest.fixture +def toy_plugin(tmp_path, monkeypatch): + import oellm.contrib as contrib_pkg + from oellm import registry + + plug = tmp_path / "toyplug" + plug.mkdir() + (plug / "__init__.py").write_text("") + (plug / "suite.py").write_text(SUITE_SRC) + monkeypatch.setattr( + contrib_pkg, "__path__", list(contrib_pkg.__path__) + [str(tmp_path)] + ) + registry._discover.cache_clear() + yield + registry._discover.cache_clear() + sys.modules.pop("oellm.contrib.toyplug.suite", None) + sys.modules.pop("oellm.contrib.toyplug", None) + + +class TestPluginProtocolConformance: + def test_discovery_and_engine_task_name(self, toy_plugin): + from oellm.registry import get_all_task_groups, get_suite + + assert get_suite("toy_suite").SUITE_NAME == "toy_suite" + merged = get_all_task_groups() + group = merged["task_groups"]["toy-task"] + # engine_task_name override lands in the task entry (jobs.csv name) + assert group["tasks"][0]["task"] == "toy_task_engine" + # ... and keys the task_metrics mapping consistently + assert merged["task_metrics"]["toy_task_engine"] == "toy_score" + + def test_lmms_adapter_override_consulted_first(self, toy_plugin): + from oellm.constants import detect_lmms_model_type + + assert detect_lmms_model_type("org/Toy-VLM-7B") == "toy_adapter" + + def test_model_flags_via_adapter_in_runner(self, toy_plugin): + from oellm.constants import EvaluationJob + from oellm.runner import EvalRunner + + job = EvaluationJob( + model_path="ToyModel-1B", + task_path="toy_task_engine", + n_shot=0, + eval_suite="toy_suite", + ) + assert EvalRunner().resolve_suite(job) == "toy_suite:toy_backend" + + def test_run_output_collected_via_parse_results(self, toy_plugin, tmp_path): + from oellm.registry import get_suite + from oellm.results import collect_results + + results_dir = tmp_path / "run" / "results" + results_dir.mkdir(parents=True) + get_suite("toy_suite").run( + model_path="ToyModel-1B", + task="toy_task_engine", + n_shot=0, + output_path=results_dir / "out.json", + model_flags="toy_backend", + env={}, + ) + out_csv = tmp_path / "run" / "eval.csv" + collect_results(str(tmp_path / "run"), str(out_csv)) + rows = list(csv.DictReader(open(out_csv))) + assert [(r["task"], r["metric_name"], float(r["performance"])) for r in rows] == [ + ("toy_task_engine", "toy_score", 0.5) + ] + + def test_core_api_version_marker(self): + from oellm.core import CORE_API_VERSION + + assert CORE_API_VERSION == "1.0" + + +class TestTabFactTask: + def test_group_wired_and_metric_mapped(self): + from oellm.results import _load_task_metrics + from oellm.task_groups import _expand_task_groups + + expanded = _expand_task_groups(["tabular-tabfact"]) + assert [(r.task, r.n_shot, r.suite) for r in expanded] == [ + ("tabfact", 0, "lm-eval-harness") + ] + assert _load_task_metrics()["tabfact"] == "acc" + + def test_prompt_serializes_table(self): + tabfact_utils = _load_task_utils("tabfact") + doc = { + "table_caption": "medals", + "table_text": "nation#gold#silver\nnorway#10#8\nsweden#7#9", + "statement": "norway won 10 gold medals", + "label": 1, + } + text = tabfact_utils.doc_to_text(doc) + assert "nation | gold | silver" in text + assert "norway | 10 | 8" in text + assert "Statement: norway won 10 gold medals" in text + assert text.endswith("Answer:") + assert json.dumps(doc) # doc stays JSON-serializable + + +class TestTimeSeriesExamTask: + def test_group_wired_and_metric_mapped(self): + from oellm.results import _load_task_metrics + from oellm.task_groups import _expand_task_groups + + expanded = _expand_task_groups(["timeseries-tsexam"]) + assert [(r.task, r.n_shot, r.suite) for r in expanded] == [ + ("timeseriesexam", 0, "lm-eval-harness") + ] + assert _load_task_metrics()["timeseriesexam"] == "acc" + + def test_prompt_subsamples_and_letters(self): + ts_utils = _load_task_utils("timeseriesexam") + doc = { + "question": "Do the two series share a distribution?", + "options": ["No, they differ", "Yes, they match"], + "answer": "Yes, they match", + "ts1": [float(i) for i in range(1000)], + "ts2": [1.0, 2.0, 3.0], + } + text = ts_utils.doc_to_text(doc) + assert "Series A (1000 points, uniformly subsampled to 128)" in text + assert "Series B (3 points" in text + assert "A. No, they differ" in text and "B. Yes, they match" in text + assert text.endswith("Answer:") + # subsample really capped the series + first_block = text.split("Series B")[0] + assert first_block.count(",") <= 130 + assert ts_utils.doc_to_choice(doc) == [" A", " B"] + assert ts_utils.doc_to_target(doc) == 1 diff --git a/tests/test_quantization_and_timeout.py b/tests/test_quantization_and_timeout.py index 8584986d..f7575e93 100644 --- a/tests/test_quantization_and_timeout.py +++ b/tests/test_quantization_and_timeout.py @@ -119,6 +119,13 @@ def test_row_timeout_recorded_in_provenance(self, tmp_path, monkeypatch): ) assert prov["row_timeout"] == "2h" + def test_submitter_recorded(self, tmp_path, monkeypatch): + monkeypatch.setenv("OELLM_SUBMITTED_BY", "ci-bot") + _, prov = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert prov["submitted_by"] == "ci-bot" + class TestEngineVersionProvenance: def test_versions_probed_from_venv(self, tmp_path, monkeypatch): @@ -177,3 +184,34 @@ def test_skip_checks_skips_probing(self, tmp_path, monkeypatch): tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 ) assert prov["engine_versions"] == {} + + +class TestAdapterRenderedModelArgs: + """The engine --model_args strings are rendered by DefaultHFAdapter and + trust_remote_code genuinely governs eval time.""" + + def test_trc_true_renders_like_before(self, tmp_path, monkeypatch): + sbatch, prov = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert 'pretrained="$model_path",trust_remote_code=True' in sbatch + assert "trust_remote_code=True,pretrained=$model_path" in sbatch + assert "model_name=$model_path,trust_remote_code=True," in sbatch + assert 'LM_EVAL_TRC="1"' in sbatch + assert prov["trust_remote_code"] is True + assert "lm_eval" in prov["engine_model_args"] + + def test_trc_false_disables_eval_time_trust(self, tmp_path, monkeypatch): + sbatch, prov = _schedule( + tmp_path, + monkeypatch, + models="org/m", + tasks="hellaswag", + n_shot=0, + trust_remote_code=False, + ) + assert "trust_remote_code=True" not in sbatch + assert 'pretrained="$model_path",trust_remote_code=False' in sbatch + assert 'LM_EVAL_TRC=""' in sbatch + assert "model_name=$model_path,batch_size" in sbatch + assert prov["trust_remote_code"] is False diff --git a/tests/test_regiondial_bench.py b/tests/test_regiondial_bench.py index a6768920..86e89ed5 100644 --- a/tests/test_regiondial_bench.py +++ b/tests/test_regiondial_bench.py @@ -193,7 +193,7 @@ def _sample( } if round is not None: d["round"] = round - return json.dumps(d) + return d class TestGIoU: @@ -211,27 +211,27 @@ def test_name(self, metric): def test_perfect_overlap(self, metric): s = _sample(100, 100) - assert metric.compute([s], [""]) == pytest.approx(1.0) + assert metric.compute([s]) == pytest.approx(1.0) def test_zero_overlap(self, metric): s = _sample(0, 200) - assert metric.compute([s], [""]) == pytest.approx(0.0) + assert metric.compute([s]) == pytest.approx(0.0) def test_partial_overlap(self, metric): s = _sample(25, 175) - assert metric.compute([s], [""]) == pytest.approx(25 / 175, abs=1e-4) + assert metric.compute([s]) == pytest.approx(25 / 175, abs=1e-4) def test_mean_over_multiple_samples(self, metric): perfect = _sample(100, 100) zero = _sample(0, 200) - score = metric.compute([perfect, zero], ["", ""]) + score = metric.compute([perfect, zero]) assert score == pytest.approx(0.5) def test_empty_input(self, metric): - assert metric.compute([], []) == pytest.approx(0.0) + assert metric.compute([]) == pytest.approx(0.0) def test_null_sample(self, metric): - score = metric.compute(["null"], [""]) + score = metric.compute([None]) assert score == pytest.approx(0.0) @@ -250,11 +250,11 @@ def test_name(self, metric): def test_perfect_overlap(self, metric): s = _sample(100, 100) - assert metric.compute([s], [""]) == pytest.approx(1.0) + assert metric.compute([s]) == pytest.approx(1.0) def test_zero_overlap(self, metric): s = _sample(0, 200) - assert metric.compute([s], [""]) == pytest.approx(0.0) + assert metric.compute([s]) == pytest.approx(0.0) def test_cumulative_formula_differs_from_giou(self, metric): from oellm.contrib.regiondial_bench.metrics import GIoU @@ -263,15 +263,14 @@ def test_cumulative_formula_differs_from_giou(self, metric): s1 = _sample(100, 100) s2 = _sample(50, 200) preds = [s1, s2] - refs = ["", ""] - ciou_val = metric.compute(preds, refs) # (100+50)/(100+200) = 0.5 - giou_val = giou.compute(preds, refs) # (1.0+0.25)/2 = 0.625 + ciou_val = metric.compute(preds) # (100+50)/(100+200) = 0.5 + giou_val = giou.compute(preds) # (1.0+0.25)/2 = 0.625 assert ciou_val == pytest.approx(0.5) assert giou_val == pytest.approx(0.625) assert ciou_val != pytest.approx(giou_val) def test_empty_input(self, metric): - assert metric.compute([], []) == pytest.approx(0.0) + assert metric.compute([]) == pytest.approx(0.0) class TestBboxAP: @@ -289,19 +288,19 @@ def test_name(self, metric): def test_all_correct(self, metric): s = _sample(100, 100, bbox_iou=0.9) - assert metric.compute([s, s], ["", ""]) == pytest.approx(1.0) + assert metric.compute([s, s]) == pytest.approx(1.0) def test_none_correct(self, metric): s = _sample(10, 200, bbox_iou=0.3) - assert metric.compute([s], [""]) == pytest.approx(0.0) + assert metric.compute([s]) == pytest.approx(0.0) def test_threshold_at_half(self, metric): above = _sample(80, 100, bbox_iou=0.6) below = _sample(10, 100, bbox_iou=0.4) - assert metric.compute([above, below], ["", ""]) == pytest.approx(0.5) + assert metric.compute([above, below]) == pytest.approx(0.5) def test_empty_input(self, metric): - assert metric.compute([], []) == pytest.approx(0.0) + assert metric.compute([]) == pytest.approx(0.0) class TestPassRate: @@ -325,14 +324,14 @@ def test_all_pass(self): s = _sample(100, 100) pr = PassRate(0.5) - assert pr.compute([s, s], ["", ""]) == pytest.approx(1.0) + assert pr.compute([s, s]) == pytest.approx(1.0) def test_none_pass(self): from oellm.contrib.regiondial_bench.metrics import PassRate s = _sample(0, 100) pr = PassRate(0.3) - assert pr.compute([s], [""]) == pytest.approx(0.0) + assert pr.compute([s]) == pytest.approx(0.0) def test_half_pass(self): from oellm.contrib.regiondial_bench.metrics import PassRate @@ -340,7 +339,7 @@ def test_half_pass(self): perfect = _sample(100, 100) zero = _sample(0, 100) pr = PassRate(0.5) - score = pr.compute([perfect, zero], ["", ""]) + score = pr.compute([perfect, zero]) assert score == pytest.approx(0.5) def test_invalid_threshold_raises(self): @@ -356,7 +355,7 @@ def test_invalid_threshold_raises(self): def test_empty_input(self): from oellm.contrib.regiondial_bench.metrics import PassRate - assert PassRate(0.5).compute([], []) == pytest.approx(0.0) + assert PassRate(0.5).compute([]) == pytest.approx(0.0) # --------------------------------------------------------------------------- From 4181e38c1d337b00623298019291ccfe81947f66 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Wed, 22 Jul 2026 11:22:09 +0200 Subject: [PATCH 31/44] [Base] README: document new modalities, operational flags, and fix inconsistencies (#32) Documentation-only. Brings the README up to date with the last two platform PRs and fixes inconsistencies found in a doc audit. --- README.md | 57 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index cb2aea97..eba67ca6 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Collect results** and check for missing evaluations: `oellm-eval collect` - **Diagnose your environment** (cluster vars, HF cache, venv engines): `oellm-eval doctor` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading -- **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Jupiter, Snellius) +- **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Jupiter, Snellius, UFAL) - **Image evaluation** via lmms-eval (VQAv2, MMBench, MMMU, ChartQA, DocVQA, TextVQA, OCRBench, OCRBench v2, MathVista, MathVision, MMStar, AI2D, RealWorldQA, MME, MME-RealWorld, SEED-Bench) - **Video evaluation** via lmms-eval (VideoMMMU, EgoSchema, VideoMME, ActivityNet-QA, LongVideoBench) - **Audio evaluation** via lmms-eval (LibriSpeech, FLEURS, GigaSpeech, TED-LIUM, WenetSpeech, CoVoST2, VocalSound, MuChoMusic) +- **Tabular & time-series evaluation** via custom lm-eval tasks (TabFact, TimeSeriesExam) - **Plugin system** for contributing custom benchmarks without touching core code - **Automatic building and deployment of containers** @@ -23,7 +24,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across | `oellm-eval eval --config eval.yaml` | Same as `schedule`, driven by a YAML config file; CLI flags override the file | | `oellm-eval collect ` | Aggregate result JSONs into `eval_results.csv` + `.json` + `.md`; `--check` writes a re-schedulable CSV of missing jobs | | `oellm-eval list-tasks` | Show every task group, its engine, task count, and n-shot settings | -| `oellm-eval compare ` | Diff two collected `results.json` files task by task | +| `oellm-eval compare ` | Diff two collected results (files or run directories) per model × task × n-shot × metric | | `oellm-eval doctor` | Diagnose the environment: cluster detection, env vars, HF cache, venv engines | ## Quick Start @@ -45,11 +46,11 @@ oellm-eval schedule \ oellm-eval schedule \ --models "llava-hf/llava-1.5-7b-hf" \ --task-groups "image-vqa" \ - --venv-path ~/elliot-venv + --venv-path /path/to/.venv ``` This will automatically: -- Detect your current HPC cluster (Leonardo, LUMI, JURECA, Jupiter, or Snellius) +- Detect your current HPC cluster (Leonardo, LUMI, JURECA, Jupiter, Snellius, or UFAL) - Download and cache the specified models - Pre-download datasets for known tasks (see warning below) - Generate and submit a SLURM job array with appropriate cluster-specific resources and using containers built for this cluster @@ -136,6 +137,15 @@ The lmms-eval adapter class (`llava_hf`, `llava_onevision`, `qwen2_5_vl`, etc.) Audio tasks also run through lmms-eval — use the general venv from [docs/VENV.md](docs/VENV.md) (the `[audio]` extra adds the audio decoding helpers, but lmms-eval itself must be installed per that guide). Judge-model groups (AIR-Bench chat, Alpaca-Audio, OpenHermes, WavCaps) need `OPENAI_API_KEY` on the compute node — scheduling refuses without it unless you pass `--allow-missing-judge`. The HPC Singularity image must include `ffmpeg` for non-WAV decode. +### Tabular & Time Series + +| Group | Benchmark | Engine | +|---|---|---| +| `tabular-tabfact` | TabFact — is a statement entailed or refuted by a Wikipedia table (few-shot from the train split) | lm-eval (custom task) | +| `timeseries-tsexam` | TimeSeriesExam — MCQ time-series understanding: trend, seasonality, anomalies, causality (strictly 0-shot) | lm-eval (custom task) | + +Both are custom lm-eval task definitions shipped in [`custom_lm_eval_tasks/`](oellm/resources/custom_lm_eval_tasks/) — no plugin needed; see path 2 in the [Contributing Guide](oellm/contrib/CONTRIBUTING.md). TabFact's dataset is script-based, so schedule it with `--trust-remote-code`. + ### Custom Benchmarks (contrib) Community-contributed benchmarks that run outside the standard evaluation engines. See the [contrib registry](oellm/contrib/README.md) for the full list. @@ -145,19 +155,25 @@ Community-contributed benchmarks that run outside the standard evaluation engine oellm-eval schedule \ --models "llava-hf/llava-1.5-7b-hf" \ --task-groups "image-vqa" \ - --venv-path ~/elliot-venv + --venv-path /path/to/.venv # Run all 5 video benchmarks oellm-eval schedule \ --models "lmms-lab/llava-onevision-7b" \ --task-groups "video-understanding" \ - --venv-path ~/elliot-venv + --venv-path /path/to/.venv # Mix image and text benchmarks in one submission oellm-eval schedule \ --models "llava-hf/llava-1.5-7b-hf" \ --task-groups "image-mmbench,open-sci-0.01" \ - --venv-path ~/elliot-venv + --venv-path /path/to/.venv + +# Tabular + time-series benchmarks (custom lm-eval tasks) +oellm-eval schedule \ + --models "meta-llama/Llama-3.1-8B-Instruct" \ + --task-groups "tabular-tabfact,timeseries-tsexam" \ + --venv-path /path/to/.venv --trust-remote-code # Use multiple task groups or a super group oellm-eval schedule --models "model-name" --task-groups "belebele-eu-5-shot,global-mmlu-eu" @@ -276,7 +292,7 @@ override these defaults: BATCH_SIZE=8 oellm-eval schedule \ --models "model-name" \ --task-groups "belebele-eu-cf" \ - --venv-path .venv + --venv-path /path/to/.venv ``` If you need full manual control over all model args, set `MODEL_ARGS`, @@ -284,9 +300,23 @@ for example: ```bash MODEL_ARGS='batch_size=8' oellm-eval schedule \ - --models "model-name" --task-groups "belebele-eu-cf" --venv-path .venv + --models "model-name" --task-groups "belebele-eu-cf" --venv-path /path/to/.venv ``` +## Quantized Evaluation & Per-Row Timeouts + +```bash +# bitsandbytes 4-bit / 8-bit loading (lm-eval, lmms-eval, evalchemy; flags are mutually exclusive) +oellm-eval schedule --models "model-name" --task-groups "open-sci-0.01" --load-in-4bit + +# Bound each evaluation row's wall clock: a hung engine fails one row (exit 124) +# instead of consuming the whole job slice; `collect --check` re-schedules just that row +ROW_TIMEOUT=30m oellm-eval schedule --models "model-name" --task-groups "open-sci-0.01" +``` + +Quantization is recorded in the run's provenance. Rows on engines that cannot +honor it run at full precision — the scheduler lists them at submission time. + ## ⚠️ Dataset Pre-Download Warning **Datasets are only automatically pre-downloaded for tasks defined in [`task-groups.yaml`](oellm/resources/task-groups.yaml).** @@ -297,7 +327,7 @@ If you use custom tasks via `--tasks` that are not in the task groups registry, ## Collecting Results -After evaluations complete, collect results into a CSV. `collect` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs: +After evaluations complete, collect results into a CSV. `collect` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs. Alongside the CSV/Markdown tables it writes `eval_results.json` — a versioned envelope that embeds each run's provenance (engine versions, model revisions, quantization, submitter), which is also the ingestion format for the ELLIOT evaluation dashboard: ``` output/ @@ -387,7 +417,7 @@ oellm-eval schedule \ --models "EleutherAI/pythia-160m" \ --tasks "gsm8k" \ --n-shot 0 \ - --venv-path .venv + --venv-path /path/to/.venv ``` Later, we will add recommendation for a project-wide setting to share tools and models. @@ -457,10 +487,11 @@ uv run oellm-eval schedule --models "EleutherAI/pythia-160m" --task-groups "open ## Contributing Custom Benchmarks -ELLIOT supports two paths for adding benchmarks: +ELLIOT supports three paths for adding benchmarks: 1. **Benchmark already in lm-eval / lighteval / lmms-eval** -- add a YAML entry to [`task-groups.yaml`](oellm/resources/task-groups.yaml) -2. **Fully custom benchmark** -- drop a contrib plugin into [`oellm/contrib/`](oellm/contrib/) +2. **Custom lm-eval task, no plugin** -- a task YAML plus optional prompt helpers in [`custom_lm_eval_tasks/`](oellm/resources/custom_lm_eval_tasks/); TabFact and TimeSeriesExam are the reference implementations +3. **Fully custom benchmark** -- drop a contrib plugin into [`oellm/contrib/`](oellm/contrib/) See the [Contributing Guide](oellm/contrib/CONTRIBUTING.md) for step-by-step instructions. From 27d1bf86b89b52221375fbb9efadbe4213f223c5 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 3 Aug 2026 17:25:06 +0200 Subject: [PATCH 32/44] [Base] Require lm-eval>=0.4.12 in the text extra * version bug fix * ruff fix --- oellm/contrib/CONTRIBUTING.md | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/oellm/contrib/CONTRIBUTING.md b/oellm/contrib/CONTRIBUTING.md index eccb4abb..33f25649 100644 --- a/oellm/contrib/CONTRIBUTING.md +++ b/oellm/contrib/CONTRIBUTING.md @@ -128,7 +128,7 @@ class MyTask(BaseTask): # Optional @property def task_group_name(self) -> str: - return "my-benchmark" # default: name with _ replaced by - + return "my-benchmark" # default: name with _ replaced by - @property def description(self) -> str: @@ -208,6 +208,7 @@ CLUSTER_ENV_VARS = ["MY_DATA_DIR"] def detect_model_flags(model_path: str) -> str | None: from oellm.contrib.my_suite.adapter import MyModelAdapter + return MyModelAdapter(model_path).to_contrib_flags() diff --git a/pyproject.toml b/pyproject.toml index fac1eaa8..e9ba8387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dev = [ # (datasets <4 vs >=4 conflict) and is installed separately via # ``uv tool install`` — see docs/VENV.md. text = [ - "lm-eval", + "lm-eval>=0.4.12", "torch", "transformers", "accelerate", From 31771feed1d0b822177d037ec0be1701bdf2c2ec Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 10 Aug 2026 09:13:03 +0200 Subject: [PATCH 33/44] [Base] Add missing latex2sympy2_extended dependency for PolyMath --- containers/jupiter.def | 1 + containers/jureca.def | 1 + containers/leonardo.def | 1 + containers/lumi.def | 1 + containers/slurm-ci.def | 1 + containers/snellius.def | 1 + pyproject.toml | 1 + 7 files changed, 7 insertions(+) diff --git a/containers/jupiter.def b/containers/jupiter.def index ec806d8c..349f2a32 100644 --- a/containers/jupiter.def +++ b/containers/jupiter.def @@ -14,6 +14,7 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 uv --version uv pip install --system --break-system-packages lm-eval==0.4.12 \ + latex2sympy2_extended \ wandb sentencepiece tiktoken accelerate # lighteval as isolated tool (avoids dependency conflicts) diff --git a/containers/jureca.def b/containers/jureca.def index ff579501..d59a99f8 100644 --- a/containers/jureca.def +++ b/containers/jureca.def @@ -14,6 +14,7 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ lm-eval==0.4.12 \ + latex2sympy2_extended \ transformers \ "datasets<4.0.0" \ wandb \ diff --git a/containers/leonardo.def b/containers/leonardo.def index 1e7f6fa6..4b69f86b 100644 --- a/containers/leonardo.def +++ b/containers/leonardo.def @@ -14,6 +14,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ lm-eval==0.4.12 \ + latex2sympy2_extended \ transformers \ "datasets<4.0.0" \ wandb \ diff --git a/containers/lumi.def b/containers/lumi.def index 14d527ce..14e856fb 100644 --- a/containers/lumi.def +++ b/containers/lumi.def @@ -14,6 +14,7 @@ From: rocm/pytorch:rocm6.4.1_ubuntu24.04_py3.12_pytorch_release_2.7.1 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ lm-eval==0.4.12 \ + latex2sympy2_extended \ transformers \ "datasets<4.0.0" \ wandb \ diff --git a/containers/slurm-ci.def b/containers/slurm-ci.def index 1b3851bc..ad1e3145 100644 --- a/containers/slurm-ci.def +++ b/containers/slurm-ci.def @@ -14,6 +14,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ lm-eval==0.4.12 \ + latex2sympy2_extended \ transformers \ "datasets<4.0.0" \ wandb \ diff --git a/containers/snellius.def b/containers/snellius.def index 3dd4e5e9..7b572fe2 100644 --- a/containers/snellius.def +++ b/containers/snellius.def @@ -14,6 +14,7 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 # lm-eval and dependencies (system Python) uv pip install --system --break-system-packages \ lm-eval==0.4.12 \ + latex2sympy2_extended \ transformers \ "datasets<4.0.0" \ wandb \ diff --git a/pyproject.toml b/pyproject.toml index e9ba8387..eeca3f9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dev = [ # ``uv tool install`` — see docs/VENV.md. text = [ "lm-eval>=0.4.12", + "latex2sympy2_extended", # required by the polymath tasks "torch", "transformers", "accelerate", From ea3bcb2071b9e1edc244ec8afcc0402f5dda25d8 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 24 Aug 2026 09:42:17 +0200 Subject: [PATCH 34/44] [Contrib][Image] Add spurious-robustness contrib plugin (ImageNet, CelebA, UrbanCars) Add spurious-robustness contrib plugin (ImageNet, CelebA, UrbanCars) --- oellm/contrib/README.md | 21 + oellm/contrib/regiondial_bench/suite.py | 13 +- oellm/contrib/spurious_robustness/README.md | 177 +++++++ oellm/contrib/spurious_robustness/__init__.py | 1 + oellm/contrib/spurious_robustness/adapter.py | 43 ++ oellm/contrib/spurious_robustness/datasets.py | 207 ++++++++ oellm/contrib/spurious_robustness/metrics.py | 81 +++ oellm/contrib/spurious_robustness/prompts.py | 50 ++ oellm/contrib/spurious_robustness/runner.py | 117 +++++ oellm/contrib/spurious_robustness/suite.py | 208 ++++++++ oellm/contrib/spurious_robustness/task.py | 76 +++ oellm/contrib/spurious_robustness/zeroshot.py | 118 +++++ oellm/contrib/spurious_urbancars/README.md | 108 ++++ oellm/contrib/spurious_urbancars/__init__.py | 1 + oellm/contrib/spurious_urbancars/suite.py | 114 +++++ oellm/contrib/spurious_urbancars/task.py | 43 ++ oellm/results.py | 6 + pyproject.toml | 10 + tests/test_plugin_protocol.py | 69 +++ tests/test_regiondial_bench.py | 64 +++ tests/test_spurious_robustness.py | 462 ++++++++++++++++++ 21 files changed, 1987 insertions(+), 2 deletions(-) create mode 100644 oellm/contrib/spurious_robustness/README.md create mode 100644 oellm/contrib/spurious_robustness/__init__.py create mode 100644 oellm/contrib/spurious_robustness/adapter.py create mode 100644 oellm/contrib/spurious_robustness/datasets.py create mode 100644 oellm/contrib/spurious_robustness/metrics.py create mode 100644 oellm/contrib/spurious_robustness/prompts.py create mode 100644 oellm/contrib/spurious_robustness/runner.py create mode 100644 oellm/contrib/spurious_robustness/suite.py create mode 100644 oellm/contrib/spurious_robustness/task.py create mode 100644 oellm/contrib/spurious_robustness/zeroshot.py create mode 100644 oellm/contrib/spurious_urbancars/README.md create mode 100644 oellm/contrib/spurious_urbancars/__init__.py create mode 100644 oellm/contrib/spurious_urbancars/suite.py create mode 100644 oellm/contrib/spurious_urbancars/task.py create mode 100644 tests/test_spurious_robustness.py diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md index a25f5628..92e95d99 100644 --- a/oellm/contrib/README.md +++ b/oellm/contrib/README.md @@ -10,6 +10,8 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). |---|---|---|---|---| | RegionDial-Bench | `regiondial-bench` | Multi-round region grounding and segmentation on RefCOCOg and RefCOCO+. Evaluates robustness to error accumulation across dialogue turns. | [arXiv:2602.03733](https://arxiv.org/abs/2602.03733) | [lmsdss/RegionReasoner](https://github.com/lmsdss/RegionReasoner) | | AudioBench | `audio-audiobench` (+ `-asr` / `-st` / `-reasoning`) | 27 judge-free audio tasks — ASR (WER), speech translation (BLEU), spoken reasoning, AudioCaps captioning — scored with AudioBench's own normalisers for paper-comparable numbers. | [arXiv:2406.16020](https://arxiv.org/abs/2406.16020) | [AudioLLMs/AudioBench](https://github.com/AudioLLMs/AudioBench) | +| Spurious robustness | `spurious-robustness` (+ `-imagenet` / `-celeba`) | Zero-shot robustness to spurious correlations for OpenCLIP dual encoders: ImageNet (no spurious attribute, top-1) and CelebA (gender, 4 groups, worst-group). | [ACM MM 2026](https://github.com/gsarridis/vlm-spurious-robustness) | [gsarridis/vlm-spurious-robustness](https://github.com/gsarridis/vlm-spurious-robustness) | +| UrbanCars | `spurious-urbancars` | Two-attribute spurious robustness (background + co-occurring object, 8 groups, worst-group). Separate suite so its required data directory is checked before submission. | [ACM MM 2026](https://github.com/gsarridis/vlm-spurious-robustness) | [gsarridis/vlm-spurious-robustness](https://github.com/gsarridis/vlm-spurious-robustness) | ### RegionDial-Bench @@ -36,3 +38,22 @@ oellm-eval schedule \ ``` Requires cluster-specific setup (`AUDIOBENCH_DIR` pointing at an AudioBench clone, a dedicated venv). Only the model families AudioBench itself supports can be evaluated (Qwen2-Audio, SALMONN, Whisper, …). See the full [AudioBench README](audiobench/README.md) for prerequisites and the supported-model table. + +### Spurious robustness + +**Metrics:** `worst_group_accuracy` (CelebA, UrbanCars), `top1_accuracy` (ImageNet), plus `avg_accuracy`, `accuracy_gap` and per-group accuracies + +```bash +oellm-eval schedule \ + --models laion/CLIP-ViT-B-32-laion2B-s34B-b79K \ + --task-groups spurious-robustness,spurious-urbancars \ + --venv-path ~/spurious-venv +``` + +Evaluates **OpenCLIP-family dual encoders**, not generative VLMs: an image is assigned to the class whose text embedding is nearest. Verified metric-for-metric against the reference implementation. + +ImageNet and CelebA are staged automatically from the Hub. ImageNet is gated and CelebA is non-commercial research only — both accepted per researcher under their own account. + +UrbanCars ships as the separate `spurious_urbancars` suite because it is the only one needing a cluster-local tree: `CLUSTER_ENV_VARS` applies to every task in a suite, so bundling it would fail ImageNet and CelebA wherever no UrbanCars data exists. Split out, `URBANCARS_DATA_DIR` is validated by the login-node pre-flight before submission. It has no Hub source — see the [UrbanCars README](spurious_urbancars/README.md) for why it must be built. + +See the full [spurious-robustness README](spurious_robustness/README.md) for group definitions, the prompting policy, and how to read a worst-group score. diff --git a/oellm/contrib/regiondial_bench/suite.py b/oellm/contrib/regiondial_bench/suite.py index 67e03bb1..652066af 100644 --- a/oellm/contrib/regiondial_bench/suite.py +++ b/oellm/contrib/regiondial_bench/suite.py @@ -337,7 +337,7 @@ def _aggregate_shards( Computes: - Aggregate metrics across all rounds: gIoU, cIoU, bbox_AP, pass_rate_* - - Per-round metrics (R1–R7): gIoU_R1..R7, bbox_AP_R1..R7 + - The same metrics per round (R1–R7), plus ``n_samples_R*`` When *expected_samples* is given, raises if the aggregated sample count differs — this catches shards that silently evaluated a partial (or @@ -409,9 +409,18 @@ def _aggregate_shards( rnd = image_turn_counter[image_id] rounds_map[rnd].append(sample_dict) - per_round_metrics = [GIoU(), BboxAP()] + per_round_metrics = [ + GIoU(), + CIoU(), + BboxAP(), + PassRate(0.3), + PassRate(0.5), + PassRate(0.7), + PassRate(0.9), + ] for rnd in sorted(rounds_map): rnd_samples = rounds_map[rnd] + metrics[f"n_samples_R{rnd}"] = len(rnd_samples) for m in per_round_metrics: val = m.compute(rnd_samples) metrics[f"{m.name}_R{rnd}"] = val diff --git a/oellm/contrib/spurious_robustness/README.md b/oellm/contrib/spurious_robustness/README.md new file mode 100644 index 00000000..ef747018 --- /dev/null +++ b/oellm/contrib/spurious_robustness/README.md @@ -0,0 +1,177 @@ +# Spurious robustness (OpenCLIP) + +Zero-shot robustness to spurious correlations, replicating Sarridis et al., +*Scaling Vision-Language Models Fails to Mitigate Bias* (ACM MM 2026). +Reference implementation: (MIT). + +| Task | Spurious attributes | Pinned metric | Groups | +|---|---|---|---| +| `spurious_imagenet` | none | `top1_accuracy` | 1 | +| `spurious_celeba` | one (gender) | `worst_group_accuracy` | 4 | +| `spurious_urbancars` | two (background, co-occurring object) | `worst_group_accuracy` | 8 | + +## What this suite evaluates + +CLIP-family **dual encoders**, not generative VLMs. An image is assigned to the +class whose text embedding is nearest in cosine similarity. That is the paper's +protocol, and it is why this is a contrib suite: neither lm-eval nor lmms-eval +can score by embedding similarity, and lm-eval's multimodal backends raise on +loglikelihood requests carrying an image, so likelihood-ranked multiple choice +is not an alternative either. + +Scores from this suite are comparable with the paper's. They are **not** +comparable with numbers obtained by prompting a generative VLM to name a class +on the same images. + +## Group definitions + +The worst-group number is the **minimum accuracy over the groups below**, and +over nothing else. Average accuracy is over all images, not the mean of the +per-group accuracies — the groups are heavily imbalanced, so the two differ. + +**CelebA — 4 groups.** Label is hair colour (`Blond_Hair`), spurious attribute +is gender (`Male`). Evaluated on the official CelebA **test** split (19,962 +images). + +| group | images | share | +|---|---:|---:| +| `non-blonde_female` | 9,767 | 48.93% | +| `non-blonde_male` | 7,535 | 37.75% | +| `blonde_female` | 2,480 | 12.42% | +| `blonde_male` | **180** | **0.90%** | + +The minimum is almost always set by `blonde_male`, the rarest combination — 180 +images, under 1% of the split. A worst-group number therefore rests on a small +sample, and `--limit` runs frequently drop the group entirely. + +**UrbanCars — 8 groups.** Label is car type; background and co-occurring object +are both spurious. Groups are the 2x2x2 product, read from the directory names: + +``` +obj={urban|country}, bg={urban|country}, co={urban|country} +``` + +The minimum is set by the groups where both shortcuts contradict the label +(`obj=urban, bg=country, co=country` and its mirror). + +**ImageNet — 1 group.** No spurious attribute, so worst-group accuracy equals +top-1 by construction. It is reported for uniformity; `top1_accuracy` is the +pinned metric. + +A group that contributes **zero samples** is absent from the minimum rather than +scored 0.0 — an unobserved group has no accuracy to be worst, and scoring it as +a total failure would make every subsampled run report 0.0. Runs that cover +fewer groups than the benchmark defines log a warning and record `n_groups`, +because a missing group can only make the minimum look better. + +## Prompting and scoring policy + +Fixed for every model; changing any of it makes scores incomparable. + +- **CelebA / UrbanCars** score a class as the **maximum** cosine similarity over + that class's prompts. Non-blonde is enumerated as concrete hair colours rather + than a negation, which CLIP text encoders handle poorly. The UrbanCars prompts + name car subtypes so that neither spurious attribute ever appears in the + prompt — otherwise the prompt would leak the shortcut being measured. +- **ImageNet** uses OpenCLIP's `IMAGENET_CLASSNAMES` under all 80 + `OPENAI_IMAGENET_TEMPLATES`, **averaged** per class (`build_zero_shot_classifier`). + +The max/average split between benchmarks is deliberate and must not be unified. +Exact prompt strings are in `prompts.py` and are frozen. + +## Interpreting a score + +| | random | majority class | always one class (worst-group) | +|---|---|---|---| +| ImageNet (1000-way) | 0.10% | 0.10% | — | +| CelebA (2-way) | ~50% | **86.67%** (always non-blonde) | **0%** | +| UrbanCars (2-way) | 50% | 50% | **0%** | + +The last column is the point of the benchmark: a model that always answers +"non-blonde" scores 86.67% average accuracy on CelebA and **0% worst-group**, +because both blonde groups score zero. A large gap between average and +worst-group means the model is riding the spurious attribute. Read the two +numbers together — an average accuracy near 87% on CelebA is exactly what a +model that has learned nothing but the class prior would produce. + +On ImageNet both baselines are 0.10% because the validation split is exactly +balanced at 50 images per class, so a score in the low single digits is +near-chance rather than a weak result. + +For reference, `laion/CLIP-ViT-B-32-laion2B-s34B-b79K` on the full CelebA test +split scores 89.58% average and 80.00% worst-group (`blonde_male`) — above the +majority-class baseline on both counts. + +## Parity with the reference implementation + +This suite is checked against the reference by running *its* benchmark classes +on identical inputs in the same process and comparing every metric. + +**CelebA — exact, on the full 19,962-image test split.** All eleven shared +metrics match bit-for-bit with `laion/CLIP-ViT-B-32-laion2B-s34B-b79K` +(worst-group 0.7944444444444444 on `blonde_male`, average 0.8964532611962729). +Label and group assignment were separately verified element-wise across all +19,962 images. + +**UrbanCars — exact, on a real eight-directory tree.** All eight per-group +accuracies, the average, the worst-group value and the worst group itself match +the reference. Only `.jpg` files are samples: the generator writes a +`_mask.png` and a `_co_occur_obj_mask.png` beside every composited image, and +those masks are segmentation output, not photographs. Fixtures include them so +file selection is exercised. + +**ImageNet — exact, on a synthetic 1000-synset tree.** Top-1, top-5 and image +count all match. The mapping is the part that matters here: the reference maps +synset to class index through its own `data/imagenet_synsets.txt`, while this +suite relies on ascending synset directory order. That file is sorted, so the +two agree — checked rather than assumed, since a mismatch would silently +scramble all 1000 labels. Images were spread across the whole index range so a +mis-ordered mapping could not pass unnoticed. + +One finding worth keeping in mind when changing this code: encoding must run +under `autocast` because the reference does. Without it, embeddings shift by +~1e-2 and the argmax flips on borderline samples. That moved CelebA worst-group +by 0.56 points — a single image out of 180. Because the rarest group holds only +180 images, worst-group accuracy has ~0.6-point granularity and is far more +sensitive to numerical drift than average accuracy is. Expect small movement +between CPU and CUDA runs for the same reason. + +## Data + +| Dataset | Source | Status | +|---|---|---| +| ImageNet | `ILSVRC/imagenet-1k`, validation | **Gated** — terms accepted per researcher under their own HF account, not by the institution. Or set `IMAGENET_VAL_DIR` to a local ImageFolder tree. | +| CelebA | `tpremoli/CelebA-attrs` | Ungated parquet mirror. CelebA itself is **non-commercial research only**, accepted per researcher. | +| UrbanCars | `URBANCARS_DATA_DIR` | **Not published.** Composited from Stanford Cars + Places + LVIS via [Whac-A-Mole](https://github.com/facebookresearch/Whac-A-Mole) at `bg-0.5_co_occur_obj-0.5`. | + +Two sourcing traps, both load-bearing: + +- The CelebA mirror stores attributes in the original **−1/+1** encoding, not the + 0/1 that `torchvision.datasets.CelebA` produces. Testing `== 0` for a negative + class matches nothing and yields a silently empty group. +- The mirror's split **names are swapped** relative to official CelebA. The + official test split (19,962 images) is published here as `validation`; the + split named `test` is the official validation partition (19,867 images). + +Compute nodes run with `HF_HUB_OFFLINE=1`. ImageNet and CelebA are declared as +dataset specs and pre-staged on the login node. UrbanCars is a local tree and is +never fetched, so it must already exist on the cluster filesystem. + +## Cluster setup + +Add to `clusters.yaml` for any cluster that runs UrbanCars: + +```yaml +URBANCARS_DATA_DIR: "/path/to/urbancars/bg-0.5_co_occur_obj-0.5/test" +``` + +Neither variable is listed in `CLUSTER_ENV_VARS`: the dispatcher treats that +list as required for *every* task in the suite, so listing them would break +CelebA and ImageNet runs on clusters without an UrbanCars tree. `run()` raises +with the same guidance when the variable is actually needed. + +## Install + +```bash +uv pip install '.[spurious-robustness]' +``` diff --git a/oellm/contrib/spurious_robustness/__init__.py b/oellm/contrib/spurious_robustness/__init__.py new file mode 100644 index 00000000..35fabf8f --- /dev/null +++ b/oellm/contrib/spurious_robustness/__init__.py @@ -0,0 +1 @@ +"""Zero-shot robustness to spurious correlations for OpenCLIP-family models.""" diff --git a/oellm/contrib/spurious_robustness/adapter.py b/oellm/contrib/spurious_robustness/adapter.py new file mode 100644 index 00000000..4fd2aa6d --- /dev/null +++ b/oellm/contrib/spurious_robustness/adapter.py @@ -0,0 +1,43 @@ +"""Model adapter for OpenCLIP dual encoders.""" + +from __future__ import annotations + +import os + +from oellm.core.base_model_adapter import BaseModelAdapter + + +class OpenClipAdapter(BaseModelAdapter): + """Resolves a model path into an OpenCLIP model specifier. + + OpenCLIP loads Hub checkpoints through an ``hf-hub:`` prefix and local + checkpoints through a filesystem path. A bare repo id such as + ``laion/CLIP-ViT-B-32-laion2B-s34B-b79K`` is therefore prefixed, while an + existing directory is passed through untouched. + """ + + def __init__(self, model_path: str) -> None: + self._path = model_path + + @property + def model_path(self) -> str: + return self._path + + def to_open_clip_spec(self) -> str: + if os.path.isdir(self._path) or self._path.startswith("hf-hub:"): + return self._path + return f"hf-hub:{self._path}" + + def to_lm_eval_args(self) -> str: + # Present only to satisfy the adapter interface: this suite never runs + # through lm-eval, because scoring is embedding similarity rather than + # token likelihood. + return f"pretrained={self._path}" + + def to_lmms_eval_args(self) -> str: + return f"pretrained={self._path}" + + def to_contrib_flags(self) -> str | None: + # One backend (open_clip) serves every checkpoint, so there is no + # model-type routing to do. + return None diff --git a/oellm/contrib/spurious_robustness/datasets.py b/oellm/contrib/spurious_robustness/datasets.py new file mode 100644 index 00000000..7403a246 --- /dev/null +++ b/oellm/contrib/spurious_robustness/datasets.py @@ -0,0 +1,207 @@ +"""Dataset loading and group assignment for the three benchmarks. + +Every loader is a generator of ``(images, labels, groups)`` batches so that a +50k-image split never has to be held in memory at once. ``groups[i]`` is the +subgroup string for sample ``i``; this is the only place where the spurious +attributes are still available, so the group must be attached here. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator +from itertools import product +from pathlib import Path + +Batch = tuple[list, list[int], list[str]] + +# CelebA: hair colour is the label, gender is the spurious attribute. +CELEBA_CLASS_NAMES = ("blonde", "non-blonde") # index 0 / 1, matching CELEBA_PROMPTS +_CELEBA_GENDER = {0: "female", 1: "male"} + +# UrbanCars: car type is the label; background and co-occurring object are both +# spurious. The eight subgroup directory names are the 2x2x2 product. +URBANCARS_CLASS_NAMES = ("urban", "country") # index 0 / 1, matching URBANCARS_PROMPTS +_URBANCARS_ATTRIBUTES = ("urban", "country") + +# The official CelebA test split (19,962 images) is the one the paper evaluates. +# In this mirror it is published under the name "validation" — the repo's +# "test" split is the official *validation* partition (19,867 images). Using the +# split named "test" would silently evaluate the wrong 19,867 images. +CELEBA_REPO = "tpremoli/CelebA-attrs" +CELEBA_SPLIT = "validation" + +IMAGENET_REPO = "ILSVRC/imagenet-1k" +IMAGENET_SPLIT = "validation" + +IMAGE_EXTENSIONS = (".jpg", ".jpeg", ".png", ".bmp", ".JPEG") + +URBANCARS_EXTENSIONS = (".jpg",) + + +def _batched(iterable, size: int): + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == size: + yield batch + batch = [] + if batch: + yield batch + + +def celeba_label_and_group(blond_attr: int, male_attr: int) -> tuple[int, str]: + """Map raw CelebA attribute values to (class index, group name). + + The mirror stores attributes in CelebA's original -1/+1 encoding, not the + 0/1 that ``torchvision.datasets.CelebA`` converts to. Testing ``== 0`` for a + negative class therefore matches nothing and silently produces an empty + group, so both attributes are read as ``== 1`` and negated explicitly. + """ + is_blonde = int(blond_attr == 1) + is_male = int(male_attr == 1) + # Class index 0 is "blonde"; the attribute is 1 when the hair *is* blonde. + return ( + 1 - is_blonde, + f"{CELEBA_CLASS_NAMES[1 - is_blonde]}_{_CELEBA_GENDER[is_male]}", + ) + + +def load_celeba(limit: int | None = None, batch_size: int = 32) -> Iterator[Batch]: + """CelebA official test split, grouped by hair colour x gender.""" + from datasets import load_dataset + + ds = load_dataset(CELEBA_REPO, split=CELEBA_SPLIT) + if limit is not None: + ds = ds.select(range(min(limit, len(ds)))) + + for rows in _batched(ds, batch_size): + images, labels, groups = [], [], [] + for row in rows: + label, group = celeba_label_and_group(row["Blond_Hair"], row["Male"]) + labels.append(label) + groups.append(group) + images.append(row["image"]) + yield images, labels, groups + + +def load_imagenet( + data_dir: str | None = None, limit: int | None = None, batch_size: int = 32 +) -> Iterator[Batch]: + """ILSVRC-2012 validation images. + + ImageNet has no spurious attribute, so every sample lands in a single group + named ``all`` and worst-group accuracy degenerates to top-1 by construction. + + Reads a local ImageFolder tree when *data_dir* is given, otherwise the HF + parquet copy (which is gated — the operator must have accepted its terms). + """ + if data_dir: + yield from _load_imagenet_folder(data_dir, limit, batch_size) + return + + from datasets import load_dataset + + ds = load_dataset(IMAGENET_REPO, split=IMAGENET_SPLIT) + if limit is not None: + ds = ds.select(range(min(limit, len(ds)))) + + for rows in _batched(ds, batch_size): + images = [r["image"] for r in rows] + labels = [int(r["label"]) for r in rows] + yield images, labels, ["all"] * len(rows) + + +def _load_imagenet_folder( + data_dir: str, limit: int | None, batch_size: int +) -> Iterator[Batch]: + """``//*.JPEG``. + + Standard ImageNet class indices are assigned in ascending synset order + (``n01440764`` -> 0), which is also the order of OpenCLIP's + ``IMAGENET_CLASSNAMES``. Sorting the synset directories therefore reproduces + the canonical index without needing a separate mapping file. + """ + root = Path(data_dir) + synsets = sorted(p.name for p in root.iterdir() if p.is_dir()) + if len(synsets) != 1000: + raise ValueError( + f"expected 1000 synset directories under {data_dir!r}, found {len(synsets)}" + ) + + def samples(): + for idx, synset in enumerate(synsets): + for fname in sorted(os.listdir(root / synset)): + if fname.endswith(IMAGE_EXTENSIONS): + yield str(root / synset / fname), idx + + stream = samples() + seen = 0 + for rows in _batched(stream, batch_size): + if limit is not None and seen >= limit: + return + if limit is not None: + rows = rows[: limit - seen] + seen += len(rows) + yield [p for p, _ in rows], [i for _, i in rows], ["all"] * len(rows) + + +def urbancars_subgroup_dirs(data_root: str) -> dict[str, str]: + """Map subgroup directory name -> path, for the subgroups that exist.""" + found = {} + for obj, bg, co in product(_URBANCARS_ATTRIBUTES, repeat=3): + name = f"obj-{obj}_bg-{bg}_co_occur_obj-{co}" + path = os.path.join(data_root, name) + if os.path.isdir(path): + found[name] = path + return found + + +def urbancars_group_label(dirname: str) -> tuple[str, int]: + """``obj-urban_bg-country_co_occur_obj-country`` -> (readable group, class index).""" + parts = dirname.split("_") + obj = parts[0].removeprefix("obj-") + bg = parts[1].removeprefix("bg-") + co = parts[-1].removeprefix("obj-") + if obj not in URBANCARS_CLASS_NAMES: + raise ValueError(f"unrecognised subgroup directory: {dirname}") + return f"obj={obj}, bg={bg}, co={co}", URBANCARS_CLASS_NAMES.index(obj) + + +def load_urbancars( + data_root: str, limit: int | None = None, batch_size: int = 32 +) -> Iterator[Batch]: + """UrbanCars test split laid out as eight subgroup directories. + + The label comes from the directory name alone, so the layout *is* the + ground truth: a renamed directory silently relabels its images. The caller + is responsible for checking that all eight subgroups were found — see + ``suite.run``. + """ + subgroups = urbancars_subgroup_dirs(data_root) + if not subgroups: + raise FileNotFoundError( + f"no UrbanCars subgroup directories under {data_root!r}; expected " + "obj-_bg-_co_occur_obj-" + ) + + def samples(): + for dirname in sorted(subgroups): + group, label = urbancars_group_label(dirname) + for fname in sorted(os.listdir(subgroups[dirname])): + if fname.endswith(URBANCARS_EXTENSIONS): + yield os.path.join(subgroups[dirname], fname), label, group + + stream = samples() + seen = 0 + for rows in _batched(stream, batch_size): + if limit is not None and seen >= limit: + return + if limit is not None: + rows = rows[: limit - seen] + seen += len(rows) + yield ( + [p for p, _, _ in rows], + [lbl for _, lbl, _ in rows], + [g for _, _, g in rows], + ) diff --git a/oellm/contrib/spurious_robustness/metrics.py b/oellm/contrib/spurious_robustness/metrics.py new file mode 100644 index 00000000..3d2e692a --- /dev/null +++ b/oellm/contrib/spurious_robustness/metrics.py @@ -0,0 +1,81 @@ +"""Metrics for the spurious-robustness benchmarks. + +A *sample record* is a dict:: + + {"correct": bool, "group": str} + +``group`` is the subgroup the sample belongs to — label crossed with every +spurious attribute. The group is attached at data-loading time, where the +attributes are still available, because by the time predictions exist the +attributes are gone. See ``datasets.py`` for the group strings each benchmark +produces and ``README.md`` for what they mean. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from oellm.core.base_metric import BaseMetric + + +def _valid(samples: Sequence[Any]) -> list[dict]: + """Keep only well-formed records. + + Malformed entries are dropped rather than scored as incorrect: they mean the + inference step failed to produce a record at all, and silently converting + that into a wrong answer would understate a group's accuracy without any + signal that data was lost. + """ + return [ + s + for s in samples + if isinstance(s, dict) and "correct" in s and s.get("group") is not None + ] + + +class AverageAccuracy(BaseMetric): + """Accuracy over all samples, ignoring group membership.""" + + @property + def name(self) -> str: + return "avg_accuracy" + + def compute(self, samples: Sequence[Any]) -> float: + records = _valid(samples) + if not records: + return 0.0 + return sum(bool(r["correct"]) for r in records) / len(records) + + +class WorstGroupAccuracy(BaseMetric): + """Minimum per-group accuracy across the groups present in *samples*. + + A group with no samples is absent from the minimum rather than scored 0.0 — + an unobserved group has no accuracy to be worst. This matters whenever a run + is subsampled: with ``--limit`` the rare groups may vanish entirely, and + treating them as zero would report a worst-group of 0.0 that says nothing + about the model. Use :func:`zeroshot.group_metrics` when the per-group + counts are needed to check that coverage was complete. + + With a single group this degenerates to plain accuracy, which is the correct + reading: ImageNet has no spurious attribute and therefore one group. + """ + + @property + def name(self) -> str: + return "worst_group_accuracy" + + def compute(self, samples: Sequence[Any]) -> float: + records = _valid(samples) + if not records: + return 0.0 + + totals: dict[str, int] = {} + hits: dict[str, int] = {} + for r in records: + g = str(r["group"]) + totals[g] = totals.get(g, 0) + 1 + hits[g] = hits.get(g, 0) + bool(r["correct"]) + + return min(hits[g] / totals[g] for g in totals) diff --git a/oellm/contrib/spurious_robustness/prompts.py b/oellm/contrib/spurious_robustness/prompts.py new file mode 100644 index 00000000..596bbe24 --- /dev/null +++ b/oellm/contrib/spurious_robustness/prompts.py @@ -0,0 +1,50 @@ +"""Class prompts for the two spurious-correlation benchmarks. + +The wording is reproduced verbatim from the reference implementation +(https://github.com/gsarridis/vlm-spurious-robustness, MIT licensed) because it +is the wording that produced the published numbers. Changing any string here +silently makes our scores incomparable with the paper's, so treat these as +frozen: a new phrasing belongs in a new task, not in an edit to these lists. + +ImageNet deliberately has no entry. It is classified with OpenCLIP's built-in +``IMAGENET_CLASSNAMES`` under all 80 ``OPENAI_IMAGENET_TEMPLATES``, which is the +standard Radford et al. (2021) protocol and is imported directly rather than +copied. +""" + +from __future__ import annotations + +# UrbanCars: urban vs. country car type. +# +# The car subtypes stand in for the class names so that neither spurious +# attribute (background, co-occurring object) ever appears in the prompt — +# otherwise the prompt would leak the very shortcut the benchmark measures. +URBANCARS_PROMPTS: dict[str, list[str]] = { + "urban": ["a photograph of a compact, sports, sedan car"], + "country": ["a photograph of a truck, jeep, pickup car"], +} + +# CelebA: blonde vs. non-blonde hair. +# +# Non-blonde is enumerated as concrete hair colours instead of a negation, +# which CLIP-style text encoders handle poorly ("not blonde" embeds close to +# "blonde"). Scoring takes the maximum over a class's prompts, not the mean — +# see ``zeroshot.class_scores``. +CELEBA_PROMPTS: dict[str, list[str]] = { + "blonde": [ + "a photo of a person with blonde hair", + "a photo of a person with light blonde hair", + "a photo of a person with golden hair", + "a photo of a person with platinum blonde hair", + ], + "non-blonde": [ + "a photo of a person with dark hair", + "a photo of a person with black hair", + "a photo of a person with brown hair", + "a photo of a brunette person", + "a photo of a person with red hair", + "a photo of a person with grey hair", + "a photo of a bald person", + "a photo of a person with auburn hair", + ], +} diff --git a/oellm/contrib/spurious_robustness/runner.py b/oellm/contrib/spurious_robustness/runner.py new file mode 100644 index 00000000..c1b35b76 --- /dev/null +++ b/oellm/contrib/spurious_robustness/runner.py @@ -0,0 +1,117 @@ +"""Inference helpers shared by the spurious-robustness suites. + +UrbanCars ships as a separate suite because it is the only task that needs a +cluster-local data directory, and ``CLUSTER_ENV_VARS`` is validated for every +task in a suite: declaring ``URBANCARS_DATA_DIR`` alongside ImageNet and CelebA +would fail those two on any cluster without an UrbanCars tree. Splitting the +suites lets the login-node pre-flight check the variable only when UrbanCars is +actually scheduled, instead of surfacing the problem inside SLURM hours later. + +Both suites share the scoring code here so the two cannot drift apart. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def resolve_device() -> str: + import torch + + return "cuda" if torch.cuda.is_available() else "cpu" + + +def load_model(model_path: str, device: str): + import open_clip + + from oellm.contrib.spurious_robustness.adapter import OpenClipAdapter + + spec = OpenClipAdapter(model_path).to_open_clip_spec() + logger.info("Loading OpenCLIP model %s on %s", spec, device) + model, _, preprocess = open_clip.create_model_and_transforms(spec, device=device) + model.eval() + tokenizer = open_clip.get_tokenizer(spec) + return model, preprocess, tokenizer + + +def read_limit(env: dict[str, str]) -> int | None: + raw = env.get("LIMIT", "").strip() + return int(raw) if raw else None + + +def run_grouped(model, preprocess, tokenizer, device, batches, prompts, class_names): + """Score a grouped benchmark (max-over-prompts, then worst-group).""" + from oellm.contrib.spurious_robustness.zeroshot import ( + class_scores, + encode_images, + encode_texts, + group_metrics, + predict, + ) + + prompt_features = [ + encode_texts(model, tokenizer, prompts[c], device) for c in class_names + ] + + predictions: list[int] = [] + labels: list[int] = [] + groups: list[str] = [] + for images, batch_labels, batch_groups in batches: + features = encode_images(model, preprocess, images, device) + predictions.extend(predict(class_scores(features, prompt_features)).tolist()) + labels.extend(batch_labels) + groups.extend(batch_groups) + + metrics = group_metrics(predictions, labels, groups) + per_group = metrics.pop("group_accuracies") + metrics.pop("group_counts") + for group, acc in sorted(per_group.items()): + metrics[f"acc_{group}"] = acc + return metrics + + +def warn_on_partial_coverage(task: str, metrics: dict, expected: int | None) -> None: + """Worst-group is a minimum, so a missing group can only flatter the score.""" + observed = metrics.get("n_groups") + if expected is not None and observed != expected: + logger.warning( + "%s covered %s of %d groups — worst-group accuracy is a minimum over " + "the groups present, so this number is not comparable with a full run.", + task, + observed, + expected, + ) + + +def write_results( + output_path: Path, model_path: str, task: str, n_shot: int, metrics: dict +) -> None: + result_json = { + "model_name_or_path": model_path, + "results": {task: metrics}, + "configs": {task: {"num_fewshot": n_shot}}, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(result_json, f, indent=2) + logger.info("Results written to %s", output_path) + + +def parse_suite_results( + data: dict, task_prefix: str +) -> tuple[str, str, int, dict[str, float]] | None: + """Claim *data* if it carries this suite's task and metric shape.""" + results = data.get("results", {}) + for task_name, task_results in results.items(): + if not task_name.startswith(task_prefix) or not isinstance(task_results, dict): + continue + if "worst_group_accuracy" not in task_results: + continue + model_id = data.get("model_name_or_path") or data.get("model_name", "unknown") + n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) + return model_id, task_name, int(n_shot), task_results + return None diff --git a/oellm/contrib/spurious_robustness/suite.py b/oellm/contrib/spurious_robustness/suite.py new file mode 100644 index 00000000..64511a09 --- /dev/null +++ b/oellm/contrib/spurious_robustness/suite.py @@ -0,0 +1,208 @@ +"""Spurious-robustness contrib suite — plugin protocol implementation. + +Replicates the evaluation of Sarridis et al., "Scaling Vision-Language Models +Fails to Mitigate Bias" (ACM MM 2026), whose reference implementation is at +https://github.com/gsarridis/vlm-spurious-robustness (MIT). + +Why this is a contrib suite rather than an lm-eval task: classification here is +image/text *embedding similarity* on an OpenCLIP dual encoder — the image is +assigned to the class whose text embedding is nearest. Neither lm-eval nor +lmms-eval has a similarity-scoring path, and lm-eval's multimodal backends raise +on loglikelihood requests that carry an image, so likelihood-ranked multiple +choice is not an alternative. The benchmark brings its own inference and its own +metric, which is exactly what this plugin interface is for. + +Consequence worth stating plainly: this suite evaluates CLIP-family dual +encoders, not the generative VLMs the rest of the platform runs. Scores are +comparable with the paper, and are not comparable with a generatively prompted +model's answers on the same images. + +This suite covers the two benchmarks whose data comes from the Hub. UrbanCars +lives in the sibling ``spurious_urbancars`` suite because it needs a +cluster-local directory: ``CLUSTER_ENV_VARS`` is validated for every task in a +suite, so declaring its path here would fail ImageNet and CelebA on any cluster +without an UrbanCars tree. Schedule both with +``--task-groups spurious-robustness,spurious-urbancars``. + +Cluster setup +------------- +``IMAGENET_VAL_DIR`` + Optional. An ILSVRC-2012 validation ImageFolder tree to read instead of the + gated Hub copy. Not required, so it is deliberately absent from + ``CLUSTER_ENV_VARS`` — the Hub copy is the default path. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +SUITE_NAME = "spurious_robustness" + +# Nothing here is mandatory: both datasets are pre-staged from the Hub. +CLUSTER_ENV_VARS: list[str] = [] + +from oellm.contrib.spurious_robustness.task import ( # noqa: E402 + SpuriousCelebATask, + SpuriousImageNetTask, +) + +_TASKS = (SpuriousImageNetTask(), SpuriousCelebATask()) + +_group_kwargs = {"suite": SUITE_NAME, "n_shots": [0]} + + +def _task_entry(task) -> dict: + entry: dict = {"task": task.engine_task_name} + if task.dataset_specs: + entry["dataset"] = task.dataset_specs[0].repo_id + return entry + + +TASK_GROUPS: dict = { + "task_metrics": {t.engine_task_name: t.primary_metric for t in _TASKS}, + "task_groups": { + "spurious-robustness": { + **_group_kwargs, + "description": ( + "Zero-shot robustness to spurious correlations for OpenCLIP " + "models: ImageNet and CelebA. Add spurious-urbancars for the " + "two-attribute benchmark." + ), + "tasks": [_task_entry(t) for t in _TASKS], + }, + **{ + t.task_group_name: { + **_group_kwargs, + "description": t.description, + "tasks": [_task_entry(t)], + } + for t in _TASKS + }, + }, +} + +_EXPECTED_GROUPS = {"spurious_celeba": 4} + + +def detect_model_flags(model_path: str) -> str | None: + """Delegate to OpenClipAdapter.to_contrib_flags().""" + from oellm.contrib.spurious_robustness.adapter import OpenClipAdapter + + return OpenClipAdapter(model_path).to_contrib_flags() + + +def _run_imagenet(model, preprocess, tokenizer, device, limit, env) -> dict: + """1000-way zero-shot top-1/top-5, the Radford et al. (2021) protocol. + + The classifier averages all 80 OpenAI templates per class — unlike CelebA and + UrbanCars, which take the maximum over their prompts. + """ + import torch + from open_clip import ( + IMAGENET_CLASSNAMES, + OPENAI_IMAGENET_TEMPLATES, + build_zero_shot_classifier, + ) + + from oellm.contrib.spurious_robustness.datasets import load_imagenet + from oellm.contrib.spurious_robustness.zeroshot import encode_images + + classifier = build_zero_shot_classifier( + model, + tokenizer=tokenizer, + classnames=IMAGENET_CLASSNAMES, + templates=OPENAI_IMAGENET_TEMPLATES, + num_classes_per_batch=10, + device=device, + use_tqdm=False, + ) + + top1 = top5 = total = 0 + for images, labels, _ in load_imagenet(env.get("IMAGENET_VAL_DIR") or None, limit): + features = encode_images(model, preprocess, images, device) + logits = 100.0 * features.to(device) @ classifier + targets = torch.tensor(labels, device=device) + _, ranked = logits.topk(5, dim=-1) + hits = ranked == targets.view(-1, 1) + top1 += hits[:, :1].sum().item() + top5 += hits.sum().item() + total += len(labels) + + if not total: + raise RuntimeError("ImageNet evaluation produced no samples") + + return { + "top1_accuracy": top1 / total, + "top5_accuracy": top5 / total, + # One group by construction: no spurious attribute to split on. + "worst_group_accuracy": top1 / total, + "n_groups": 1, + "total_images": total, + } + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Evaluate *task* and write a lmms-eval-compatible JSON to *output_path*.""" + from oellm.contrib.spurious_robustness.datasets import ( + CELEBA_CLASS_NAMES, + load_celeba, + ) + from oellm.contrib.spurious_robustness.prompts import CELEBA_PROMPTS + from oellm.contrib.spurious_robustness.runner import ( + load_model, + read_limit, + resolve_device, + run_grouped, + warn_on_partial_coverage, + write_results, + ) + + known = [t.engine_task_name for t in _TASKS] + if task not in known: + raise ValueError(f"Unknown task {task!r}. Expected one of: {known}") + + limit = read_limit(env) + device = resolve_device() + model, preprocess, tokenizer = load_model(model_path, device) + + if task == "spurious_imagenet": + metrics = _run_imagenet(model, preprocess, tokenizer, device, limit, env) + else: + metrics = run_grouped( + model, + preprocess, + tokenizer, + device, + load_celeba(limit), + CELEBA_PROMPTS, + CELEBA_CLASS_NAMES, + ) + + warn_on_partial_coverage(task, metrics, _EXPECTED_GROUPS.get(task)) + write_results(output_path, model_path, task, n_shot, metrics) + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Claim *data* if it is this suite's output, else return None. + + UrbanCars output belongs to the sibling suite, so it is explicitly not + claimed here — a suite that recognises a file owns its format. + """ + from oellm.contrib.spurious_robustness.runner import parse_suite_results + + for prefix in ("spurious_imagenet", "spurious_celeba"): + claimed = parse_suite_results(data, prefix) + if claimed is not None: + return claimed + return None diff --git a/oellm/contrib/spurious_robustness/task.py b/oellm/contrib/spurious_robustness/task.py new file mode 100644 index 00000000..24d9c28f --- /dev/null +++ b/oellm/contrib/spurious_robustness/task.py @@ -0,0 +1,76 @@ +"""Task definitions for the spurious-robustness benchmarks.""" + +from __future__ import annotations + +from oellm.core.base_task import BaseTask +from oellm.task_groups import DatasetSpec + +SUITE_NAME = "spurious_robustness" + + +class SpuriousImageNetTask(BaseTask): + """ImageNet: overall recognition, the no-spurious-attribute reference point.""" + + @property + def name(self) -> str: + return "spurious_imagenet" + + @property + def suite(self) -> str: + return SUITE_NAME + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def primary_metric(self) -> str: + return "top1_accuracy" + + @property + def description(self) -> str: + return ( + "ImageNet zero-shot top-1 on the ILSVRC-2012 validation split " + "(1000-way, 80 OpenAI templates averaged per class). No spurious " + "attribute — the recognition baseline the worst-group numbers are read against." + ) + + @property + def dataset_specs(self) -> list[DatasetSpec]: + return [DatasetSpec(repo_id="ILSVRC/imagenet-1k")] + + +class SpuriousCelebATask(BaseTask): + """CelebA: one spurious attribute (gender), four groups.""" + + @property + def name(self) -> str: + return "spurious_celeba" + + @property + def suite(self) -> str: + return SUITE_NAME + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def primary_metric(self) -> str: + return "worst_group_accuracy" + + @property + def description(self) -> str: + return ( + "CelebA worst-group accuracy over hair colour x gender (4 groups) " + "on the official test split. Single-attribute robustness." + ) + + @property + def dataset_specs(self) -> list[DatasetSpec]: + return [DatasetSpec(repo_id="tpremoli/CelebA-attrs")] + + +# UrbanCars lives in the sibling ``spurious_urbancars`` suite: it is the only +# task needing a cluster-local directory, and CLUSTER_ENV_VARS applies to every +# task in a suite. diff --git a/oellm/contrib/spurious_robustness/zeroshot.py b/oellm/contrib/spurious_robustness/zeroshot.py new file mode 100644 index 00000000..2394e91a --- /dev/null +++ b/oellm/contrib/spurious_robustness/zeroshot.py @@ -0,0 +1,118 @@ +"""Zero-shot inference primitives.""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, Any + +import numpy as np + +# Kept out of module scope: the base install has no torch. +if TYPE_CHECKING: + import torch + + +def encode_texts(model, tokenizer, texts: Sequence[str], device: str) -> torch.Tensor: + """L2-normalised text embeddings, ``(len(texts), D)`` on CPU.""" + import torch + import torch.nn.functional as F + + tokens = tokenizer(list(texts)).to(device) + with torch.no_grad(), torch.amp.autocast(device): + features = model.encode_text(tokens) + return F.normalize(features.float().cpu(), dim=-1) + + +def encode_images( + model, + preprocess, + images: Iterable[Any], + device: str, + batch_size: int = 32, +) -> torch.Tensor: + """L2-normalised image embeddings, ``(N, D)`` on CPU. Takes PIL images or paths.""" + import torch + import torch.nn.functional as F + from PIL import Image + + batch: list[torch.Tensor] = [] + chunks: list[torch.Tensor] = [] + + def flush() -> None: + if not batch: + return + stacked = torch.stack(batch).to(device) + with torch.no_grad(), torch.amp.autocast(device): + features = model.encode_image(stacked) + chunks.append(features.float().cpu()) + batch.clear() + + for item in images: + img = Image.open(item) if isinstance(item, str) else item + batch.append(preprocess(img.convert("RGB"))) + if len(batch) == batch_size: + flush() + flush() + + if not chunks: + return torch.empty(0) + return F.normalize(torch.cat(chunks, dim=0), dim=-1) + + +def class_scores( + image_features: torch.Tensor, prompt_features: Sequence[torch.Tensor] +) -> np.ndarray: + """Per-class similarity scores, ``(N, n_classes)``. Max over a class's prompts.""" + columns = [(image_features @ pf.T).numpy().max(axis=1) for pf in prompt_features] + return np.stack(columns, axis=-1) + + +def predict(scores: np.ndarray) -> np.ndarray: + """Predicted class index per image.""" + return np.argmax(scores, axis=-1) + + +def group_metrics( + predictions: Sequence[int], + labels: Sequence[int], + group_names: Sequence[str], +) -> dict: + """Overall, per-group and worst-group accuracy. + + ``group_names[i]`` is the subgroup of sample ``i``. Average accuracy is over + all samples, not the mean of per-group accuracies. Worst-group is the + minimum over groups present in the data. + """ + predictions_arr = np.asarray(predictions) + labels_arr = np.asarray(labels) + groups = np.asarray(group_names) + + if labels_arr.size == 0: + raise ValueError("no samples to score") + + correct = predictions_arr == labels_arr + per_group: dict[str, float] = {} + counts: dict[str, int] = {} + for name in sorted(set(groups.tolist())): + mask = groups == name + per_group[name] = float(correct[mask].mean()) + counts[name] = int(mask.sum()) + + if not per_group: + raise ValueError("no groups found") + + worst = min(per_group, key=per_group.__getitem__) + best = max(per_group, key=per_group.__getitem__) + avg = float(correct.mean()) + return { + "avg_accuracy": avg, + "worst_group_accuracy": per_group[worst], + "worst_group": worst, + "best_group_accuracy": per_group[best], + "best_group": best, + "accuracy_gap": avg - per_group[worst], + "group_accuracies": per_group, + "group_counts": counts, + "n_groups": len(per_group), + "total_images": int(labels_arr.size), + } diff --git a/oellm/contrib/spurious_urbancars/README.md b/oellm/contrib/spurious_urbancars/README.md new file mode 100644 index 00000000..f41d5a6b --- /dev/null +++ b/oellm/contrib/spurious_urbancars/README.md @@ -0,0 +1,108 @@ +# UrbanCars (OpenCLIP) + +Two-attribute spurious robustness, replicating Sarridis et al., *Scaling +Vision-Language Models Fails to Mitigate Bias* (ACM MM 2026). +Reference implementation: (MIT). + +Shares its scoring code, prompts and metric with +[`spurious_robustness`](../spurious_robustness/README.md), which covers ImageNet +and CelebA. Read that README for the prompting policy, the interpretation +baselines, and how worst-group accuracy is computed. + +**Pinned metric:** `worst_group_accuracy` (minimum over 8 groups) + +```bash +oellm-eval schedule \ + --models laion/CLIP-ViT-B-32-laion2B-s34B-b79K \ + --task-groups spurious-urbancars \ + --venv-path ~/spurious-venv +``` + +Both suites together: + +```bash +oellm-eval schedule \ + --models laion/CLIP-ViT-B-32-laion2B-s34B-b79K \ + --task-groups spurious-robustness,spurious-urbancars \ + --venv-path ~/spurious-venv +``` + +## Why this is a separate suite + +`CLUSTER_ENV_VARS` is validated for *every* task in a suite. UrbanCars is the +only benchmark of the three needing a cluster-local directory, so declaring +`URBANCARS_DATA_DIR` alongside ImageNet and CelebA would have failed those two +on any cluster without an UrbanCars tree. As its own suite, the variable is +checked by the login-node pre-flight before submission — the operator learns at +schedule time rather than after a job has been queued and started. + +## Group definitions + +Label is car type; background and co-occurring object are both spurious. The 8 +groups are the 2x2x2 product, read from the directory names: + +``` +obj={urban|country}, bg={urban|country}, co={urban|country} +``` + +The minimum is set by the groups where both shortcuts contradict the label — +`obj=urban, bg=country, co=country` and its mirror. + +A group contributing zero samples is absent from the minimum rather than scored +0.0. Runs covering fewer than 8 groups log a warning and record `n_groups`, +because a missing group can only make the minimum look better. + +Baselines: random 50%, majority class 50%, and a degenerate always-one-class +predictor scores **0% worst-group** — that is the signal the benchmark exists to +produce. + +## Data — this dataset must be built, not downloaded + +There is no published UrbanCars dataset. It does not exist on the Hugging Face +Hub, and the reference implementation only *consumes* a prebuilt tree. It is +composited by [Whac-A-Mole](https://github.com/facebookresearch/Whac-A-Mole) via +`scripts/prepare_dataset_models/create_urbancars.sh`, which requires: + +- **Stanford Cars** — `cars_train.tgz`, `cars_test.tgz`, `car_devkit.tgz` + (the original Stanford URLs have been offline since 2024; a mirror is needed) +- **COCO 2017** train + val images +- **LVIS** v1 train/val annotation JSON +- **Places365-standard**, 256x256 +- **MaskFormer** panoptic checkpoint plus a clone of its repo (needs detectron2) + +The script then runs car-mask segmentation on GPU and composites the result. +That is tens of gigabytes of input and a dependency set that cannot share this +suite's venv, which is why it is out of scope for automatic staging here. + +Once built, point the cluster at the test split: + +```yaml +URBANCARS_DATA_DIR: "/path/to/urbancars/bg-0.5_co_occur_obj-0.5/test" +``` + +Expected layout — eight directories of `*.jpg`: + +``` +obj-urban_bg-urban_co_occur_obj-urban/ +obj-urban_bg-urban_co_occur_obj-country/ +obj-urban_bg-country_co_occur_obj-urban/ +obj-urban_bg-country_co_occur_obj-country/ +obj-country_bg-urban_co_occur_obj-urban/ +obj-country_bg-urban_co_occur_obj-country/ +obj-country_bg-country_co_occur_obj-urban/ +obj-country_bg-country_co_occur_obj-country/ +``` + +The label comes from the directory name alone, so the layout *is* the ground +truth: a renamed directory silently relabels its images. + +If the built tree is uploaded to an internal Hub dataset repo, it can be staged +automatically like ImageNet and CelebA — that only needs a `dataset_specs` entry +on the task, plus a decision about hosting and redistribution, since the result +is derived from four separately-licensed sources. + +## Parity + +Verified against the reference implementation by running its +`UrbanCarsBenchmark` on an identical eight-directory tree in the same process: +all 15 shared metrics match exactly, including every per-group accuracy. diff --git a/oellm/contrib/spurious_urbancars/__init__.py b/oellm/contrib/spurious_urbancars/__init__.py new file mode 100644 index 00000000..3bc45fc0 --- /dev/null +++ b/oellm/contrib/spurious_urbancars/__init__.py @@ -0,0 +1 @@ +"""UrbanCars: two-attribute spurious robustness for OpenCLIP models.""" diff --git a/oellm/contrib/spurious_urbancars/suite.py b/oellm/contrib/spurious_urbancars/suite.py new file mode 100644 index 00000000..885b6ecb --- /dev/null +++ b/oellm/contrib/spurious_urbancars/suite.py @@ -0,0 +1,114 @@ +"""UrbanCars contrib suite — plugin protocol implementation. + +Split out from ``spurious_robustness`` so that ``URBANCARS_DATA_DIR`` can be +declared in ``CLUSTER_ENV_VARS``. That list is validated for every task in a +suite, so keeping UrbanCars alongside ImageNet and CelebA would have failed +those two on any cluster without an UrbanCars tree. As its own suite the +variable is checked by the login-node pre-flight before submission, rather than +failing inside SLURM once the job is already running. + +Scoring is shared with ``spurious_robustness`` so the two cannot drift apart. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +SUITE_NAME = "spurious_urbancars" + +# UrbanCars is composited locally (Stanford Cars + Places + LVIS via +# Whac-A-Mole) rather than published, so the tree must already exist on the +# cluster. Declaring it here gives the operator a failure at schedule time. +CLUSTER_ENV_VARS = ["URBANCARS_DATA_DIR"] + +from oellm.contrib.spurious_urbancars.task import SpuriousUrbanCarsTask # noqa: E402 + +_TASK = SpuriousUrbanCarsTask() + +TASK_GROUPS: dict = { + "task_metrics": {_TASK.engine_task_name: _TASK.primary_metric}, + "task_groups": { + _TASK.task_group_name: { + "suite": SUITE_NAME, + "n_shots": _TASK.n_shots, + "description": _TASK.description, + "tasks": [{"task": _TASK.engine_task_name}], + } + }, +} + +_EXPECTED_GROUPS = 8 + +_MISSING_DIR_HINT = ( + "URBANCARS_DATA_DIR must be set for spurious_urbancars. UrbanCars is " + "composited locally (Stanford Cars + Places + LVIS via Whac-A-Mole) rather " + "than published, so it cannot be fetched from the Hub. Point it at the " + "bg-0.5_co_occur_obj-0.5 test tree." +) + + +def detect_model_flags(model_path: str) -> str | None: + from oellm.contrib.spurious_robustness.adapter import OpenClipAdapter + + return OpenClipAdapter(model_path).to_contrib_flags() + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Evaluate UrbanCars and write a lmms-eval-compatible JSON to *output_path*.""" + from oellm.contrib.spurious_robustness.datasets import ( + URBANCARS_CLASS_NAMES, + load_urbancars, + ) + from oellm.contrib.spurious_robustness.prompts import URBANCARS_PROMPTS + from oellm.contrib.spurious_robustness.runner import ( + load_model, + read_limit, + resolve_device, + run_grouped, + warn_on_partial_coverage, + write_results, + ) + + if task != _TASK.engine_task_name: + raise ValueError(f"Unknown task {task!r}. Expected {_TASK.engine_task_name!r}.") + + # Fail on configuration before paying for a checkpoint load. The scheduler's + # pre-flight should already have caught this on the login node; this is the + # compute-node backstop for a cluster whose value points somewhere stale. + data_dir = env.get("URBANCARS_DATA_DIR", "") + if not data_dir: + raise RuntimeError(_MISSING_DIR_HINT) + + limit = read_limit(env) + device = resolve_device() + model, preprocess, tokenizer = load_model(model_path, device) + + metrics = run_grouped( + model, + preprocess, + tokenizer, + device, + load_urbancars(data_dir, limit), + URBANCARS_PROMPTS, + URBANCARS_CLASS_NAMES, + ) + warn_on_partial_coverage(task, metrics, _EXPECTED_GROUPS) + write_results(output_path, model_path, task, n_shot, metrics) + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Claim *data* if it is this suite's output, else return None.""" + from oellm.contrib.spurious_robustness.runner import parse_suite_results + + return parse_suite_results(data, "spurious_urbancars") diff --git a/oellm/contrib/spurious_urbancars/task.py b/oellm/contrib/spurious_urbancars/task.py new file mode 100644 index 00000000..b2821e86 --- /dev/null +++ b/oellm/contrib/spurious_urbancars/task.py @@ -0,0 +1,43 @@ +"""UrbanCars task definition.""" + +from __future__ import annotations + +from oellm.core.base_task import BaseTask +from oellm.task_groups import DatasetSpec + +SUITE_NAME = "spurious_urbancars" + + +class SpuriousUrbanCarsTask(BaseTask): + """UrbanCars: two spurious attributes (background, co-occurring object).""" + + @property + def name(self) -> str: + return "spurious_urbancars" + + @property + def suite(self) -> str: + return SUITE_NAME + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def primary_metric(self) -> str: + return "worst_group_accuracy" + + @property + def description(self) -> str: + return ( + "UrbanCars worst-group accuracy over car type x background x " + "co-occurring object (8 groups). Multi-attribute robustness. Reads a " + "prebuilt image tree from URBANCARS_DATA_DIR; not available on the Hub." + ) + + @property + def dataset_specs(self) -> list[DatasetSpec]: + # UrbanCars is composited locally rather than published, so there is + # nothing for the login node to pre-stage. URBANCARS_DATA_DIR supplies + # the image tree and is checked before submission via CLUSTER_ENV_VARS. + return [] diff --git a/oellm/results.py b/oellm/results.py index fc6ceda5..b539b1e2 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -63,6 +63,12 @@ "mme_cognition_score": 800.0, "mme_perception_score": 2000.0, # ── Contrib / additional 0–1 metrics ── + # Spurious-robustness suite. worst_group_accuracy is a minimum over + # subgroup accuracies, so it shares accuracy's 0–1 scale. + "worst_group_accuracy": 1.0, + "avg_accuracy": 1.0, + "top1_accuracy": 1.0, + "top5_accuracy": 1.0, "gIoU": 1.0, "meteor": 1.0, "string_match": 1.0, diff --git a/pyproject.toml b/pyproject.toml index eeca3f9b..e0abc203 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,16 @@ audiobench = [ "soundfile", "librosa", ] +# Spurious-robustness contrib plugin. Scores OpenCLIP-family dual encoders by +# image/text embedding similarity, so it needs open_clip rather than an +# lm-eval-style generative engine; it shares a venv with nothing else. +spurious-robustness = [ + "open_clip_torch>=2.24", + "torch", + "pillow", + # CelebA and ImageNet are read as HF parquet; UrbanCars is a local image tree. + "datasets<4.0.0", +] # Evalchemy reasoning suite — uses a forked lm-eval that cannot coexist # with mainline lm-eval (different fork at the same import path). Install # in its own venv: ``uv pip install '.[evalchemy]'``. Pinned versions are diff --git a/tests/test_plugin_protocol.py b/tests/test_plugin_protocol.py index 0e95f686..d42da831 100644 --- a/tests/test_plugin_protocol.py +++ b/tests/test_plugin_protocol.py @@ -256,3 +256,72 @@ def test_prompt_subsamples_and_letters(self): assert first_block.count(",") <= 130 assert ts_utils.doc_to_choice(doc) == [" A", " B"] assert ts_utils.doc_to_target(doc) == 1 + + +class TestSpuriousRobustnessSuite: + """Conformance for the OpenCLIP spurious-robustness contrib suite.""" + + def test_groups_wired_and_metrics_mapped(self): + from oellm.results import _load_task_metrics + from oellm.task_groups import _expand_task_groups + + metrics = _load_task_metrics() + assert metrics["spurious_imagenet"] == "top1_accuracy" + assert metrics["spurious_celeba"] == "worst_group_accuracy" + assert metrics["spurious_urbancars"] == "worst_group_accuracy" + + expanded = _expand_task_groups(["spurious-celeba"]) + assert [(r.task, r.n_shot, r.suite) for r in expanded] == [ + ("spurious_celeba", 0, "spurious_robustness") + ] + + # UrbanCars is a separate suite (its data dir is a required env var), so + # the combined group covers only the two Hub-staged benchmarks. + combined = _expand_task_groups(["spurious-robustness"]) + assert {r.task for r in combined} == {"spurious_imagenet", "spurious_celeba"} + + urbancars = _expand_task_groups(["spurious-urbancars"]) + assert [(r.task, r.n_shot, r.suite) for r in urbancars] == [ + ("spurious_urbancars", 0, "spurious_urbancars") + ] + + def test_pinned_metrics_have_a_native_scale(self): + """A pinned metric with no scale entry renders a blank normalized column.""" + from oellm.results import METRIC_NATIVE_SCALE + + for metric in ("top1_accuracy", "worst_group_accuracy", "avg_accuracy"): + assert METRIC_NATIVE_SCALE[metric] == 1.0 + + def test_prompt_construction_on_synthetic_doc(self): + """The class prompt sets are frozen: scores move if this test is edited.""" + from oellm.contrib.spurious_robustness.prompts import ( + CELEBA_PROMPTS, + URBANCARS_PROMPTS, + ) + + assert CELEBA_PROMPTS["blonde"][0] == "a photo of a person with blonde hair" + assert len(CELEBA_PROMPTS["blonde"]) == 4 + assert len(CELEBA_PROMPTS["non-blonde"]) == 8 + # Non-blonde is enumerated, never phrased as a negation. + assert not any("not " in p for p in CELEBA_PROMPTS["non-blonde"]) + # Neither spurious attribute may leak into the UrbanCars prompts. + for prompts in URBANCARS_PROMPTS.values(): + assert len(prompts) == 1 + assert "background" not in prompts[0] + assert "urban" not in prompts[0] and "country" not in prompts[0] + + def test_parse_results_claims_own_and_rejects_foreign(self): + from oellm.contrib.spurious_robustness import suite + + mine = { + "model_name_or_path": "hf-hub:laion/CLIP-ViT-B-32", + "results": {"spurious_celeba": {"worst_group_accuracy": 0.25}}, + "configs": {"spurious_celeba": {"num_fewshot": 0}}, + } + assert suite.parse_results(mine) == ( + "hf-hub:laion/CLIP-ViT-B-32", + "spurious_celeba", + 0, + {"worst_group_accuracy": 0.25}, + ) + assert suite.parse_results({"results": {"vqav2_val": {"vqa_score": 0.7}}}) is None diff --git a/tests/test_regiondial_bench.py b/tests/test_regiondial_bench.py index 86e89ed5..32099620 100644 --- a/tests/test_regiondial_bench.py +++ b/tests/test_regiondial_bench.py @@ -949,3 +949,67 @@ def test_failed_shard_reported_after_all_finish(self, rr_env, tmp_path): _FakeInferencePopen.returncode_for_dir = {"shard_1": 3} with pytest.raises(RuntimeError, match="shard 1 exited with code 3"): self._run(rr_env, tmp_path) + + +class TestPerRoundBreakdown: + """Every metric is reported per round, not just gIoU and bbox_AP. + + The benchmark measures error accumulation across dialogue turns, and rounds + hold very different sample counts, so the corpus-level figures alone cannot + show degradation — an overall number is dominated by the early turns. + """ + + def _write(self, tmp_path, samples): + (tmp_path / "output_0.json").write_text(json.dumps(samples)) + + def test_all_metrics_reported_per_round(self, tmp_path): + from oellm.contrib.regiondial_bench.suite import _aggregate_shards + + # One image, three turns, degrading: iou 1.0 -> 0.6 -> 0.2 + self._write( + tmp_path, + [ + {"image_id": "a", "intersection": 100, "union": 100, "bbox_iou": 1.0}, + {"image_id": "a", "intersection": 60, "union": 100, "bbox_iou": 1.0}, + {"image_id": "a", "intersection": 20, "union": 100, "bbox_iou": 0.0}, + ], + ) + m = _aggregate_shards(str(tmp_path)) + + for rnd in (1, 2, 3): + for key in ("gIoU", "cIoU", "bbox_AP"): + assert f"{key}_R{rnd}" in m, f"missing {key}_R{rnd}" + for thr in ("0.3", "0.5", "0.7", "0.9"): + assert f"pass_rate_{thr}_R{rnd}" in m + assert m[f"n_samples_R{rnd}"] == 1 + + # Hand-computed: the degradation must be visible round by round. + assert m["gIoU_R1"] == pytest.approx(1.0) + assert m["gIoU_R2"] == pytest.approx(0.6) + assert m["gIoU_R3"] == pytest.approx(0.2) + assert m["cIoU_R2"] == pytest.approx(0.6) + assert m["bbox_AP_R3"] == pytest.approx(0.0) + # pass_rate uses a strict >, so 0.6 clears 0.5 but not 0.7. + assert m["pass_rate_0.5_R2"] == pytest.approx(1.0) + assert m["pass_rate_0.7_R2"] == pytest.approx(0.0) + + def test_overall_hides_what_per_round_shows(self, tmp_path): + """The reason this breakdown exists, as an executable example.""" + from oellm.contrib.regiondial_bench.suite import _aggregate_shards + + # Round 1 has 4 samples (all pass), round 2 has 1 (fails). The overall + # pass rate is 0.8 and looks healthy; round 2 is a total failure. + samples = [ + {"image_id": f"i{i}", "intersection": 90, "union": 100, "bbox_iou": 1.0} + for i in range(4) + ] + samples.append( + {"image_id": "i0", "intersection": 0, "union": 100, "bbox_iou": 0.0} + ) + self._write(tmp_path, samples) + m = _aggregate_shards(str(tmp_path)) + + assert m["pass_rate_0.5"] == pytest.approx(0.8) + assert m["pass_rate_0.5_R1"] == pytest.approx(1.0) + assert m["pass_rate_0.5_R2"] == pytest.approx(0.0) + assert m["n_samples_R1"] == 4 and m["n_samples_R2"] == 1 diff --git a/tests/test_spurious_robustness.py b/tests/test_spurious_robustness.py new file mode 100644 index 00000000..4a977dfd --- /dev/null +++ b/tests/test_spurious_robustness.py @@ -0,0 +1,462 @@ +"""Tests for the spurious-robustness contrib suite. + +The worst-group cases carry hand-computed expected values: worst-group accuracy +is a minimum, so an off-by-one in group assignment produces a plausible-looking +number rather than an error. +""" + +from __future__ import annotations + +import pytest + +from oellm.contrib.spurious_robustness import datasets as ds +from oellm.contrib.spurious_robustness import prompts +from oellm.contrib.spurious_robustness.adapter import OpenClipAdapter +from oellm.contrib.spurious_robustness.metrics import ( + AverageAccuracy, + WorstGroupAccuracy, +) +from oellm.contrib.spurious_robustness.zeroshot import group_metrics + + +def _records(spec): + """(group, n_correct, n_total) triples -> sample records.""" + out = [] + for group, n_correct, n_total in spec: + out += [{"correct": True, "group": group}] * n_correct + out += [{"correct": False, "group": group}] * (n_total - n_correct) + return out + + +class TestWorstGroupAccuracy: + def test_hand_computed_minimum(self): + # blonde_male 1/4 = 0.25 is the minimum; blonde_female 3/4 = 0.75, + # non-blonde_male 8/10 = 0.8, non-blonde_female 9/10 = 0.9. + samples = _records( + [ + ("blonde_male", 1, 4), + ("blonde_female", 3, 4), + ("non-blonde_male", 8, 10), + ("non-blonde_female", 9, 10), + ] + ) + assert WorstGroupAccuracy().compute(samples) == pytest.approx(0.25) + + def test_average_is_over_samples_not_groups(self): + """Group means and the overall mean differ under imbalance.""" + # 1/4 + 9/10 correct = 10 of 14 samples = 0.714..., whereas the mean of + # the two group accuracies would be (0.25 + 0.9) / 2 = 0.575. + samples = _records([("rare", 1, 4), ("common", 9, 10)]) + assert AverageAccuracy().compute(samples) == pytest.approx(10 / 14) + assert AverageAccuracy().compute(samples) != pytest.approx(0.575) + + def test_single_group_degenerates_to_accuracy(self): + samples = _records([("all", 3, 4)]) + assert WorstGroupAccuracy().compute(samples) == pytest.approx(0.75) + assert AverageAccuracy().compute(samples) == pytest.approx(0.75) + + def test_group_with_zero_samples_is_absent_not_zero(self): + """An unobserved group has no accuracy, so it cannot be the minimum. + + Scoring it 0.0 would make every subsampled run report a worst-group of + zero regardless of the model. + """ + samples = _records([("present_a", 2, 2), ("present_b", 1, 2)]) + # "missing_group" contributes nothing and must not drag the result to 0. + assert WorstGroupAccuracy().compute(samples) == pytest.approx(0.5) + + def test_empty_and_malformed_input(self): + assert WorstGroupAccuracy().compute([]) == 0.0 + assert AverageAccuracy().compute([]) == 0.0 + # Records missing a group, or not dicts at all, are dropped rather than + # counted as wrong answers. + assert WorstGroupAccuracy().compute([None, "junk", {"correct": True}]) == 0.0 + mixed = [{"correct": False, "group": "g"}, None, {"correct": True}] + assert WorstGroupAccuracy().compute(mixed) == pytest.approx(0.0) + + def test_all_correct_and_all_wrong(self): + assert WorstGroupAccuracy().compute(_records([("a", 2, 2), ("b", 3, 3)])) == 1.0 + assert WorstGroupAccuracy().compute(_records([("a", 0, 2), ("b", 3, 3)])) == 0.0 + + +class TestGroupMetrics: + def test_hand_computed_full_output(self): + # a: 1/4 = 0.25 (worst), b: 3/4 = 0.75, c: 4/4 = 1.0 (best) + # overall: 8 of 12 = 0.666... + preds = [1, 0, 0, 0] + [1, 1, 1, 0] + [1, 1, 1, 1] + labels = [1, 1, 1, 1] * 3 + groups = ["a"] * 4 + ["b"] * 4 + ["c"] * 4 + + m = group_metrics(preds, labels, groups) + assert m["worst_group_accuracy"] == pytest.approx(0.25) + assert m["worst_group"] == "a" + assert m["best_group_accuracy"] == pytest.approx(1.0) + assert m["best_group"] == "c" + assert m["avg_accuracy"] == pytest.approx(8 / 12) + assert m["accuracy_gap"] == pytest.approx(8 / 12 - 0.25) + assert m["n_groups"] == 3 + assert m["total_images"] == 12 + assert m["group_counts"] == {"a": 4, "b": 4, "c": 4} + + def test_single_group(self): + m = group_metrics([1, 1, 0], [1, 1, 1], ["all"] * 3) + assert m["n_groups"] == 1 + assert m["worst_group_accuracy"] == pytest.approx(m["avg_accuracy"]) + + def test_empty_input_raises(self): + with pytest.raises(ValueError, match="no samples"): + group_metrics([], [], []) + + +class TestCelebAGrouping: + @pytest.mark.parametrize( + ("blond", "male", "expected_label", "expected_group"), + [ + (1, 1, 0, "blonde_male"), + (1, -1, 0, "blonde_female"), + (-1, 1, 1, "non-blonde_male"), + (-1, -1, 1, "non-blonde_female"), + ], + ) + def test_minus_one_plus_one_encoding( + self, blond, male, expected_label, expected_group + ): + """The mirror uses -1/+1; a `== 0` test would empty the negative groups.""" + assert ds.celeba_label_and_group(blond, male) == (expected_label, expected_group) + + def test_all_four_groups_are_reachable(self): + groups = {ds.celeba_label_and_group(b, m)[1] for b in (1, -1) for m in (1, -1)} + assert groups == { + "blonde_male", + "blonde_female", + "non-blonde_male", + "non-blonde_female", + } + + def test_official_test_split_is_the_one_named_validation(self): + """The mirror's split names are swapped relative to official CelebA.""" + assert ds.CELEBA_SPLIT == "validation" + + +class TestUrbanCarsGrouping: + def test_group_label_and_class_index(self): + assert ds.urbancars_group_label("obj-urban_bg-country_co_occur_obj-country") == ( + "obj=urban, bg=country, co=country", + 0, + ) + assert ds.urbancars_group_label("obj-country_bg-urban_co_occur_obj-urban") == ( + "obj=country, bg=urban, co=urban", + 1, + ) + + def test_unrecognised_directory_raises(self): + with pytest.raises(ValueError, match="unrecognised subgroup"): + ds.urbancars_group_label("obj-suburban_bg-urban_co_occur_obj-urban") + + def test_finds_all_eight_subgroups(self, tmp_path): + for obj in ("urban", "country"): + for bg in ("urban", "country"): + for co in ("urban", "country"): + (tmp_path / f"obj-{obj}_bg-{bg}_co_occur_obj-{co}").mkdir() + found = ds.urbancars_subgroup_dirs(str(tmp_path)) + assert len(found) == 8 + labels = {ds.urbancars_group_label(d)[0] for d in found} + assert len(labels) == 8 + + def test_partial_layout_is_reported_not_padded(self, tmp_path): + """A missing subgroup shrinks the group set; it is never invented.""" + (tmp_path / "obj-urban_bg-urban_co_occur_obj-urban").mkdir() + assert len(ds.urbancars_subgroup_dirs(str(tmp_path))) == 1 + + def test_missing_tree_raises(self, tmp_path): + with pytest.raises(FileNotFoundError, match="no UrbanCars subgroup"): + list(ds.load_urbancars(str(tmp_path))) + + +class TestUrbanCarsFileSelection: + @staticmethod + def _tree(root, n_per_group=2): + for obj in ("urban", "country"): + for bg in ("urban", "country"): + for co in ("urban", "country"): + d = root / f"obj-{obj}_bg-{bg}_co_occur_obj-{co}" + d.mkdir() + for i in range(n_per_group): + for suffix in (".jpg", "_mask.png", "_co_occur_obj_mask.png"): + (d / f"{i:03d}{suffix}").touch() + return root + + def test_mask_pngs_are_not_scored(self, tmp_path): + d = tmp_path / "obj-urban_bg-urban_co_occur_obj-urban" + d.mkdir() + for name in ("000.jpg", "000_mask.png", "000_co_occur_obj_mask.png"): + (d / name).touch() + batches = list(ds.load_urbancars(str(tmp_path))) + paths = [p for imgs, _, _ in batches for p in imgs] + assert paths == [str(d / "000.jpg")] + + def test_sample_count_matches_jpg_count_not_file_count(self, tmp_path): + self._tree(tmp_path, n_per_group=5) + batches = list(ds.load_urbancars(str(tmp_path), batch_size=64)) + paths = [p for imgs, _, _ in batches for p in imgs] + assert len(paths) == 8 * 5 + assert all(p.endswith(".jpg") for p in paths) + + def test_every_group_is_represented(self, tmp_path): + self._tree(tmp_path, n_per_group=3) + batches = list(ds.load_urbancars(str(tmp_path), batch_size=64)) + groups = {g for _, _, gs in batches for g in gs} + assert len(groups) == 8 + + +class TestReferenceParity: + def test_urbancars_prompts_are_frozen(self): + assert prompts.URBANCARS_PROMPTS == { + "urban": ["a photograph of a compact, sports, sedan car"], + "country": ["a photograph of a truck, jeep, pickup car"], + } + + def test_celeba_prompts_are_frozen(self): + assert prompts.CELEBA_PROMPTS == { + "blonde": [ + "a photo of a person with blonde hair", + "a photo of a person with light blonde hair", + "a photo of a person with golden hair", + "a photo of a person with platinum blonde hair", + ], + "non-blonde": [ + "a photo of a person with dark hair", + "a photo of a person with black hair", + "a photo of a person with brown hair", + "a photo of a brunette person", + "a photo of a person with red hair", + "a photo of a person with grey hair", + "a photo of a bald person", + "a photo of a person with auburn hair", + ], + } + + def test_class_name_order_matches_prompt_key_order(self): + assert ds.CELEBA_CLASS_NAMES == tuple(prompts.CELEBA_PROMPTS) + assert ds.URBANCARS_CLASS_NAMES == tuple(prompts.URBANCARS_PROMPTS) + + def test_urbancars_class_index_matches_reference(self): + assert ds.URBANCARS_CLASS_NAMES.index("urban") == 0 + assert ds.URBANCARS_CLASS_NAMES.index("country") == 1 + + def test_class_score_is_max_over_prompts_not_mean(self): + torch = pytest.importorskip("torch") + from oellm.contrib.spurious_robustness.zeroshot import class_scores, predict + + image_features = torch.tensor([[1.0, 0.0]]) + prompt_features = [ + torch.tensor([[0.9, 0.0], [0.0, 0.0]]), + torch.tensor([[0.5, 0.0], [0.5, 0.0]]), + ] + scores = class_scores(image_features, prompt_features) + assert scores[0][0] == pytest.approx(0.9) + assert scores[0][1] == pytest.approx(0.5) + assert predict(scores).tolist() == [0] + + +class TestTasksAndAdapter: + def test_task_properties(self): + from oellm.contrib.spurious_robustness.task import ( + SpuriousCelebATask, + SpuriousImageNetTask, + ) + from oellm.contrib.spurious_urbancars.task import SpuriousUrbanCarsTask + + imagenet, celeba, urbancars = ( + SpuriousImageNetTask(), + SpuriousCelebATask(), + SpuriousUrbanCarsTask(), + ) + assert imagenet.primary_metric == "top1_accuracy" + assert celeba.primary_metric == "worst_group_accuracy" + assert urbancars.primary_metric == "worst_group_accuracy" + for task in (imagenet, celeba, urbancars): + assert task.n_shots == [0] + assert task.description + + assert imagenet.suite == "spurious_robustness" + assert celeba.suite == "spurious_robustness" + # UrbanCars is its own suite so its data dir can be a required env var. + assert urbancars.suite == "spurious_urbancars" + + # Staged from the Hub; UrbanCars has nothing to stage. + assert imagenet.dataset_specs[0].repo_id == "ILSVRC/imagenet-1k" + assert celeba.dataset_specs[0].repo_id == "tpremoli/CelebA-attrs" + assert urbancars.dataset_specs == [] + + def test_adapter_resolves_open_clip_spec(self, tmp_path): + assert ( + OpenClipAdapter("laion/CLIP-ViT-B-32").to_open_clip_spec() + == "hf-hub:laion/CLIP-ViT-B-32" + ) + # Already-prefixed and local-directory specs pass through untouched. + assert OpenClipAdapter("hf-hub:laion/X").to_open_clip_spec() == "hf-hub:laion/X" + assert OpenClipAdapter(str(tmp_path)).to_open_clip_spec() == str(tmp_path) + + def test_hub_suite_requires_no_cluster_env_vars(self): + """Listing UrbanCars' path here would fail CelebA and ImageNet rows.""" + from oellm.contrib.spurious_robustness import suite + + assert suite.CLUSTER_ENV_VARS == [] + + def test_urbancars_suite_declares_its_data_dir(self): + """Declared so the login-node pre-flight catches it before submission.""" + from oellm.contrib.spurious_urbancars import suite + + assert suite.CLUSTER_ENV_VARS == ["URBANCARS_DATA_DIR"] + + def test_urbancars_without_data_dir_raises_with_guidance(self, tmp_path): + from oellm.contrib.spurious_urbancars import suite + + with pytest.raises(RuntimeError, match="URBANCARS_DATA_DIR"): + suite.run( + model_path="laion/CLIP-ViT-B-32", + task="spurious_urbancars", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags=None, + env={"URBANCARS_DATA_DIR": ""}, + ) + + def test_suites_do_not_claim_each_others_results(self): + """A suite that recognises a file owns its format.""" + from oellm.contrib.spurious_robustness import suite as hub_suite + from oellm.contrib.spurious_urbancars import suite as uc_suite + + uc = { + "model_name_or_path": "m", + "results": {"spurious_urbancars": {"worst_group_accuracy": 0.1}}, + } + celeba = { + "model_name_or_path": "m", + "results": {"spurious_celeba": {"worst_group_accuracy": 0.2}}, + } + assert hub_suite.parse_results(uc) is None + assert uc_suite.parse_results(celeba) is None + assert uc_suite.parse_results(uc)[1] == "spurious_urbancars" + assert hub_suite.parse_results(celeba)[1] == "spurious_celeba" + + def test_login_node_preflight_checks_only_urbancars(self, monkeypatch): + """The whole point of the split: no false failures for the Hub tasks.""" + import sys + from pathlib import Path + + from oellm.envcheck import collect_problems + + monkeypatch.delenv("URBANCARS_DATA_DIR", raising=False) + venv = str(Path(sys.prefix)) + + problems = collect_problems({"spurious_urbancars"}, venv_path=venv) + assert any("URBANCARS_DATA_DIR" in p for p in problems) + assert collect_problems({"spurious_robustness"}, venv_path=venv) == [] + + def test_unknown_task_raises(self, tmp_path): + from oellm.contrib.spurious_robustness import suite + + with pytest.raises(ValueError, match="Unknown task"): + suite.run( + model_path="laion/CLIP-ViT-B-32", + task="spurious_nonexistent", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags=None, + env={}, + ) + + +class TestSchedule: + """Render-level checks: the SLURM script and jobs.csv the scheduler emits.""" + + def _schedule(self, tmp_path, group): + import os + import sys + from pathlib import Path + from unittest.mock import patch + + from oellm.main import schedule_evals + + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="laion/CLIP-ViT-B-32-laion2B-s34B-b79K", + task_groups=group, + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + + def test_dry_run_routes_to_contrib_dispatch(self, tmp_path): + self._schedule(tmp_path, "spurious-celeba") + sbatch = list(tmp_path.glob("**/submit_evals.sbatch")) + assert len(sbatch) == 1 + assert "oellm.contrib.dispatch" in sbatch[0].read_text() + + def test_jobs_csv_carries_the_suite_and_task(self, tmp_path): + import pandas as pd + + self._schedule(tmp_path, "spurious-robustness,spurious-urbancars") + csvs = list(tmp_path.glob("**/jobs.csv")) + assert len(csvs) == 1 + df = pd.read_csv(csvs[0]) + # The frozen jobs.csv schema. + assert list(df.columns) == ["model_path", "task_path", "n_shot", "eval_suite"] + assert set(df["task_path"]) == { + "spurious_imagenet", + "spurious_celeba", + "spurious_urbancars", + } + # Both groups schedule together; each row carries its own suite so the + # dispatcher routes UrbanCars to the suite that requires its data dir. + by_task = dict(zip(df["task_path"], df["eval_suite"], strict=True)) + assert by_task["spurious_imagenet"] == "spurious_robustness" + assert by_task["spurious_celeba"] == "spurious_robustness" + assert by_task["spurious_urbancars"] == "spurious_urbancars" + assert set(df["n_shot"]) == {0} + + +class TestImageNetFolderLoader: + """The local ImageFolder path, whose class indices must match OpenCLIP's order.""" + + def _tree(self, tmp_path, n_synsets, images_per_synset=1): + # Synset ids are not contiguous in reality; what matters is that ascending + # synset order is the canonical class order. + for i in range(n_synsets): + d = tmp_path / f"n{i:08d}" + d.mkdir() + for j in range(images_per_synset): + (d / f"img_{j}.JPEG").write_bytes(b"") + return tmp_path + + def test_label_index_follows_sorted_synset_order(self, tmp_path): + root = self._tree(tmp_path, 1000) + batches = list(ds.load_imagenet(str(root), batch_size=256)) + paths = [p for b in batches for p in b[0]] + labels = [lbl for b in batches for lbl in b[1]] + groups = [g for b in batches for g in b[2]] + + assert len(labels) == 1000 + # Ascending synset directory order == ascending class index. + assert labels == sorted(labels) + assert labels[0] == 0 and labels[-1] == 999 + assert "n00000000" in paths[0] and "n00000999" in paths[-1] + # ImageNet has no spurious attribute: exactly one group. + assert set(groups) == {"all"} + + def test_limit_is_honoured(self, tmp_path): + root = self._tree(tmp_path, 1000) + labels = [lbl for b in ds.load_imagenet(str(root), limit=10) for lbl in b[1]] + assert len(labels) == 10 + + def test_wrong_class_count_raises(self, tmp_path): + """A partial tree would silently shift every class index.""" + root = self._tree(tmp_path, 3) + with pytest.raises(ValueError, match="expected 1000 synset directories"): + list(ds.load_imagenet(str(root))) From eb5eebc481ec5db6109196cfa725b144a592aab3 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Thu, 27 Aug 2026 13:43:31 +0200 Subject: [PATCH 35/44] [Base] Make lm_eval batch size configurable, explicit for local runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --batch_size auto was hardcoded for the lm_eval and evalchemy engines. lm_eval's auto search halves on CUDA OOM, so with no GPU nothing bounds it: it probes at batch 64 × prompt length and exhausts RAM. TimeSeriesExam (8 docs) reached 19.7 GB RSS and never produced a score; TabFact behaved the same. Short-prompt tasks are unaffected, which is why this went unnoticed. The template now reads ${LM_EVAL_BATCH_SIZE:-auto}, resolved by the scheduler to 8 for --local and auto on the cluster, with BATCH_SIZE overriding either — the same convention _resolve_additional_model_args already uses for lighteval, so one variable now covers all three engines. Cluster behaviour is unchanged. --- oellm/resources/template.sbatch | 5 ++- oellm/scheduler.py | 29 +++++++++++++ tests/test_batch_size.py | 76 +++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tests/test_batch_size.py diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 73f470ad..4b512446 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -26,6 +26,7 @@ export OELLM_QUANTIZATION="{quantization}" LM_EVAL_TRC="{lm_eval_trc}" VENV_PATH="{venv_path}" LM_EVAL_INCLUDE_PATH="{lm_eval_include_path}" +LM_EVAL_BATCH_SIZE="{lm_eval_batch_size}" # Compute nodes are air-gapped — every dataset must be cache-resolved. export HF_HOME=$HF_HOME @@ -180,7 +181,7 @@ do --output_path "{evals_dir}/$(openssl rand -hex 5).json" \ ${{LM_EVAL_TRC:+--trust_remote_code}} \ ${{LM_EVAL_INCLUDE_PATH:+--include_path $$LM_EVAL_INCLUDE_PATH}} \ - --batch_size auto \ + --batch_size "${{LM_EVAL_BATCH_SIZE:-auto}}" \ ${{LIMIT:+--limit $LIMIT}} rc=$? echo "----------------------------------------------------" @@ -272,7 +273,7 @@ do --model hf \ --tasks "$task_path" \ --model_args "{evalchemy_model_args}" \ - --batch_size auto \ + --batch_size "${{LM_EVAL_BATCH_SIZE:-auto}}" \ --output_path "$RESULTS_SUBDIR" \ ${{LIMIT:+--limit $LIMIT}} < /dev/null ) diff --git a/oellm/scheduler.py b/oellm/scheduler.py index c86842fb..11aea105 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -73,6 +73,34 @@ def _resolve_slurm_mem() -> str: return "96G" +def _resolve_lm_eval_batch_size(local: bool = False) -> str: + """Return the ``--batch_size`` value for the lm_eval / evalchemy engines. + + ``auto`` asks lm_eval to search for the largest batch that fits, halving + whenever CUDA raises OOM. Without a GPU that OOM never arrives, so the + search probes at ``max_batch_size`` (64) times the prompt length and can + allocate tens of GB on a long-prompt task before the machine gives out. + Local runs therefore get an explicit small batch; cluster runs keep + ``auto``. ``BATCH_SIZE`` overrides both. + """ + fallback = "8" if local else "auto" + batch_size = os.environ.get("BATCH_SIZE") + if batch_size is not None and str(batch_size).strip() != "": + batch_size_value = str(batch_size).strip() + try: + if int(batch_size_value) < 1: + raise ValueError + except ValueError: + logging.warning( + "Invalid BATCH_SIZE=%r; falling back to batch_size=%s.", + batch_size, + fallback, + ) + else: + return batch_size_value + return fallback + + def _resolve_additional_model_args(local: bool = False) -> str: """Return model args for lighteval, defaulting to an explicit batch size. @@ -654,6 +682,7 @@ def _lower_suite_only(s: str) -> str: evalchemy_model_args=evalchemy_model_args, lighteval_trc=lighteval_trc, lm_eval_trc="1" if trust_remote_code else "", + lm_eval_batch_size=_resolve_lm_eval_batch_size(local), ) # Drop optional #SBATCH directives whose env var is unset, so safe_substitute diff --git a/tests/test_batch_size.py b/tests/test_batch_size.py new file mode 100644 index 00000000..36552623 --- /dev/null +++ b/tests/test_batch_size.py @@ -0,0 +1,76 @@ +"""Tests for the lm_eval / evalchemy ``--batch_size`` selection.""" + +from unittest.mock import patch + +import pytest + +from oellm.scheduler import _resolve_lm_eval_batch_size + + +def _schedule(tmp_path, monkeypatch, **kw): + from oellm.scheduler import schedule_evals + + monkeypatch.setenv("EVAL_OUTPUT_DIR", str(tmp_path)) + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + ): + schedule_evals(dry_run=True, skip_checks=True, **kw) + return next(tmp_path.glob("**/submit_evals.sbatch")).read_text() + + +class TestResolveBatchSize: + def test_local_is_explicit_and_cluster_is_auto(self, monkeypatch): + monkeypatch.delenv("BATCH_SIZE", raising=False) + assert _resolve_lm_eval_batch_size(local=True) == "8" + assert _resolve_lm_eval_batch_size(local=False) == "auto" + + @pytest.mark.parametrize("value", ["1", "16", "64"]) + def test_env_override_wins_everywhere(self, monkeypatch, value): + monkeypatch.setenv("BATCH_SIZE", value) + assert _resolve_lm_eval_batch_size(local=True) == value + assert _resolve_lm_eval_batch_size(local=False) == value + + @pytest.mark.parametrize("value", ["0", "-4", "auto", "big", " "]) + def test_invalid_override_falls_back_to_the_default(self, monkeypatch, value): + monkeypatch.setenv("BATCH_SIZE", value) + assert _resolve_lm_eval_batch_size(local=True) == "8" + assert _resolve_lm_eval_batch_size(local=False) == "auto" + + +class TestBatchSizeReachesTheScript: + def test_both_engines_read_the_variable(self, tmp_path, monkeypatch): + monkeypatch.delenv("BATCH_SIZE", raising=False) + sbatch = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + # lm_eval and evalchemy both defer to it, and the literal is gone + assert sbatch.count('--batch_size "${LM_EVAL_BATCH_SIZE:-auto}"') == 2 + assert "--batch_size auto" not in sbatch + + def test_cluster_run_keeps_auto(self, tmp_path, monkeypatch): + monkeypatch.delenv("BATCH_SIZE", raising=False) + sbatch = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert 'LM_EVAL_BATCH_SIZE="auto"' in sbatch + + def test_local_run_pins_a_small_batch(self, tmp_path, monkeypatch): + monkeypatch.delenv("BATCH_SIZE", raising=False) + sbatch = _schedule( + tmp_path, + monkeypatch, + models="org/m", + tasks="hellaswag", + n_shot=0, + local=True, + venv_path="/tmp/venv", + ) + assert 'LM_EVAL_BATCH_SIZE="8"' in sbatch + + def test_override_is_rendered(self, tmp_path, monkeypatch): + monkeypatch.setenv("BATCH_SIZE", "4") + sbatch = _schedule( + tmp_path, monkeypatch, models="org/m", tasks="hellaswag", n_shot=0 + ) + assert 'LM_EVAL_BATCH_SIZE="4"' in sbatch From 2f40b14a3c15305f095ad2c187fa52da3f8e61ce Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Tue, 15 Sep 2026 10:56:41 +0200 Subject: [PATCH 36/44] lmms-eval: real adapter names, registry pre-flight, run-time device placement (LMMS_MODEL_ARGS), VENV.md install fix --- README.md | 4 + docs/VENV.md | 32 ++++-- oellm/constants.py | 16 +-- oellm/core/base_model_adapter.py | 6 +- oellm/envcheck.py | 75 ++++++++++++++ oellm/resources/template.sbatch | 14 ++- oellm/scheduler.py | 1 + tests/test_audio_task_groups.py | 21 ++-- tests/test_envcheck.py | 69 +++++++++++++ tests/test_lmms_adapters.py | 162 +++++++++++++++++++++++++++++++ tests/test_lmms_local_args.py | 54 +++++++++++ 11 files changed, 430 insertions(+), 24 deletions(-) create mode 100644 tests/test_lmms_adapters.py create mode 100644 tests/test_lmms_local_args.py diff --git a/README.md b/README.md index b05410b7..604f4c64 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,10 @@ oellm-eval schedule \ Results are written to `./oellm-output//results/`. +lmms-eval tasks (image, video, audio) target MPS automatically on macOS; set +`LMMS_MODEL_ARGS` (for example `device=cpu,device_map=cpu`) to override the +device arguments on any host. See [docs/VENV.md](docs/VENV.md). + **Air-gapped cluster nodes (no internet):** batch jobs set `HF_HUB_OFFLINE=1` and get `HF_HOME` from your cluster env. With `--local`, the CLI defaults `HF_HOME` to `~/.cache/huggingface` if unset and would otherwise allow Hub access—so on a compute node without network, export your real cache and offline flag before running, for example: ```bash diff --git a/docs/VENV.md b/docs/VENV.md index 148fa354..f55d3d39 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -34,12 +34,13 @@ venv against the groups you plan to run. # 1. Create venv uv venv --python 3.12 /path/to/.venv -# 2. Install lmms-eval editable from git -# (Editable is required — wheel build drops `_default_template_yaml` files.) -# Pin a known-good commit (`...lmms-eval.git@#egg=...`) so two venvs -# created on different days run the same engine — unpinned `main` drifts. -uv pip install --python /path/to/.venv/bin/python \ - -e "git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git@45c766f60b6f8c153e4c72d06ca636e2db0ebcdb#egg=lmms-eval" +# 2. Install lmms-eval editable from a clone at the pinned commit +# (Editable is required — wheel build drops `_default_template_yaml` files. +# uv refuses `-e git+…`, so clone first. Keep the commit pinned so two venvs +# created on different days run the same engine — unpinned `main` drifts.) +git clone https://github.com/EvolvingLMMs-Lab/lmms-eval.git /path/to/lmms-eval +git -C /path/to/lmms-eval checkout 45c766f60b6f8c153e4c72d06ca636e2db0ebcdb +uv pip install --python /path/to/.venv/bin/python -e /path/to/lmms-eval # 3. Install oellm-cli with engine extras uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' @@ -73,6 +74,25 @@ oellm-eval schedule \ --venv-path /path/to/.venv ``` +## Local runs without CUDA (macOS) + +lmms-eval adapters default to `device=cuda`. With `--local` on macOS the job +script targets MPS automatically (`device=mps,device_map=mps`). On any host, +`LMMS_MODEL_ARGS` (comma-separated `key=value` pairs) is appended to lmms-eval's +`--model_args` and takes precedence, for example: + +```bash +LMMS_MODEL_ARGS="device=cpu,device_map=cpu" oellm-eval schedule … --local +``` + +Several lmms-eval adapters import `decord` at module load, and there is no +macOS arm64 wheel for it, so those adapters need a `decord` build on a Mac. + +Idefics3-architecture models (SmolVLM, Idefics3) cannot be evaluated with the +pinned lmms-eval and `transformers<4.50`: no dedicated adapter exists and the +generic `huggingface` adapter loads them without a language-model head. The +scheduler refuses them at schedule time. + ## Why Multiple Install Steps? diff --git a/oellm/constants.py b/oellm/constants.py index ad70a2a9..b78190e0 100644 --- a/oellm/constants.py +++ b/oellm/constants.py @@ -15,15 +15,17 @@ class EvaluationJob: # Mapping of model path patterns to lmms-eval adapter class names. # Patterns are matched case-insensitively against the model path. # Order matters: more specific patterns must come before general ones. +# Adapter names must exist in the target venv's lmms-eval; pre-flight checks that. LMMS_MODEL_ADAPTERS: list[tuple[list[str], str]] = [ # ── Audio / speech models (must come before generic "qwen" catch-all) ── (["qwen2-audio", "qwen2_audio"], "qwen2_audio"), - (["qwen2.5-audio", "qwen2_5_audio"], "qwen2_5_audio"), - (["salmonn"], "salmonn"), - (["audio-flamingo", "audio_flamingo"], "audio_flamingo"), - (["ultravox"], "ultravox"), + (["qwen2.5-omni", "qwen2_5_omni"], "qwen2_5_omni"), + (["video-salmonn", "video_salmonn"], "video_salmonn_2"), + (["audio-flamingo-3", "audio_flamingo_3"], "audio_flamingo_3"), + (["kimi-audio", "kimi_audio"], "kimi_audio"), + (["whisper"], "whisper"), (["phi-4-multimodal", "phi_4_multimodal", "phi4-multimodal"], "phi4_multimodal"), - (["gemini-audio", "gemini_audio"], "gemini_audio"), + (["gemini"], "gemini_api"), (["gpt4o-audio", "gpt_4o_audio"], "gpt4o_audio"), # ── Vision / video models ── (["qwen2.5-vl", "qwen2_5_vl", "qwen2.5vl"], "qwen2_5_vl"), @@ -35,7 +37,9 @@ class EvaluationJob: (["llava"], "llava_hf"), (["internvideo"], "internvideo2"), (["internvl"], "internvl2"), - (["idefics"], "idefics3"), + # No idefics3/SmolVLM entry: the pinned lmms-eval has no adapter that can + # load them with transformers<4.50 (its generic one loads a headless model). + (["idefics2"], "idefics2"), (["minicpm"], "minicpm_v"), (["longva"], "longva"), (["videochat2"], "videochat2"), diff --git a/oellm/core/base_model_adapter.py b/oellm/core/base_model_adapter.py index 24896364..e1b01194 100644 --- a/oellm/core/base_model_adapter.py +++ b/oellm/core/base_model_adapter.py @@ -97,9 +97,9 @@ def to_lm_eval_args(self) -> str: return f'pretrained="{self._path}",trust_remote_code={self._trust}{self._extra}' def to_lmms_eval_args(self) -> str: - # $_lmms_extra_args is the bash-side per-family hook filled in - # template.sbatch (llava model_name workaround, qwen frame cap). - return f"pretrained={self._path},device_map=auto$_lmms_extra_args{self._extra}" + # $_lmms_extra_args is set by template.sbatch on the node: device + # placement plus per-family hooks. + return f"pretrained={self._path}$_lmms_extra_args{self._extra}" def to_evalchemy_args(self) -> str: return f"trust_remote_code={self._trust},pretrained={self._path}{self._extra}" diff --git a/oellm/envcheck.py b/oellm/envcheck.py index cb7cc39a..b5a4d9ac 100644 --- a/oellm/envcheck.py +++ b/oellm/envcheck.py @@ -20,6 +20,8 @@ from __future__ import annotations +import difflib +import json import os import shutil import subprocess @@ -133,6 +135,57 @@ def probe_import(python_bin: str | Path, module: str) -> tuple[bool, str]: return False, err_lines[-1] if err_lines else "import failed" +def lmms_adapters_from_suites(suites: set[str] | list[str]) -> set[str]: + """Adapter names carried by ``lmms_eval:`` suite strings.""" + from oellm.runner import EvalRunner + + out = set() + for s in suites: + head, sep, tail = str(s).partition(":") + if ( + sep + and tail.strip() + and EvalRunner.canonical_name(head.strip().lower()) == "lmms_eval" + ): + out.add(tail.strip()) + return out + + +_LMMS_REGISTRY_PROBE = """ +import json +import lmms_eval.models as m +names = set() +for attr in ("AVAILABLE_MODELS", "AVAILABLE_SIMPLE_MODELS", "AVAILABLE_CHAT_MODELS"): + v = getattr(m, attr, None) + if isinstance(v, dict): + names.update(v) +reg = getattr(m, "MODEL_REGISTRY_V2", None) +if reg is not None and hasattr(reg, "list_model_names"): + names.update(reg.list_model_names()) +print(json.dumps(sorted(names))) +""" + + +def lmms_registered_adapters(python_bin: str | Path) -> set[str] | None: + """Adapter names the venv's lmms-eval registers; ``None`` when unknowable.""" + try: + r = subprocess.run( + [str(python_bin), "-c", _LMMS_REGISTRY_PROBE], + capture_output=True, + text=True, + timeout=_PROBE_TIMEOUT_S, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if r.returncode != 0: + return None + try: + names = json.loads(r.stdout.strip().splitlines()[-1]) + except (ValueError, IndexError): + return None + return set(names) or None + + def _find_executable(name: str, venv_path: str | Path | None) -> str | None: """Resolve *name* from the venv's bin dir first, then PATH.""" if venv_path: @@ -160,6 +213,7 @@ def collect_problems( problems: list[str] = [] canonical = canonical_suites(suites) venv_python = Path(venv_path).expanduser() / "bin" / "python" if venv_path else None + lmms_importable = False for suite in sorted(canonical): req = _requirements_for_suite(suite) @@ -181,13 +235,17 @@ def collect_problems( # the static container_ok flag is the contract. continue + modules_ok = True for module in req.modules: ok, detail = probe_import(venv_python, module) if not ok: + modules_ok = False problems.append( f"suite '{suite}': module '{module}' is not importable in " f"venv {venv_path} ({detail}). {req.hint}" ) + if suite == "lmms_eval" and modules_ok: + lmms_importable = True for exe in req.executables: if _find_executable(exe, venv_path) is None: @@ -210,6 +268,23 @@ def collect_problems( f"filesystem. {req.hint}" ) + # LMMS_MODEL_ADAPTERS and the installed lmms-eval drift apart; an unknown + # adapter would otherwise fail on the node after the queue wait. + adapters = lmms_adapters_from_suites(suites) + if adapters and lmms_importable: + registered = lmms_registered_adapters(venv_python) + if registered is not None: + for adapter in sorted(adapters - registered): + close = difflib.get_close_matches(adapter, sorted(registered), n=3) + hint = f" Closest registered names: {', '.join(close)}." if close else "" + problems.append( + f"suite 'lmms_eval': adapter '{adapter}' (resolved from the model " + f"name) is not registered in the lmms-eval installed at " + f"{venv_path} ({len(registered)} adapters available).{hint} " + f"Fix LMMS_MODEL_ADAPTERS in oellm/constants.py or install an " + f"lmms-eval that provides it." + ) + for group in group_names or []: for module, required_version in GROUP_VERSION_PINS.get(group, ()): if venv_path is None: diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 4b512446..7681393c 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -237,8 +237,20 @@ do _lmms_adapter="${{eval_suite#*:}}" OUTPUT_JSON="{evals_dir}/$(openssl rand -hex 5).json" + # lmms-eval adapters default to device=cuda. LMMS_MODEL_ARGS (k=v,...) + # is appended verbatim; a --local run on macOS defaults to MPS. + if [ -n "${{LMMS_MODEL_ARGS:-}}" ]; then + case ",${{LMMS_MODEL_ARGS:-}}," in + *,device_map=*) _lmms_extra_args=",${{LMMS_MODEL_ARGS:-}}" ;; + *) _lmms_extra_args=",device_map=auto,${{LMMS_MODEL_ARGS:-}}" ;; + esac + elif [ "$(uname -s)" = "Darwin" ]; then + _lmms_extra_args=",device=mps,device_map=mps" + else + _lmms_extra_args=",device_map=auto" + fi + # LLaVA adapters need model_name to avoid a missing-import bug in lmms-eval - _lmms_extra_args="" if [[ "$_lmms_adapter" == "llava_onevision" || "$_lmms_adapter" == "llava_vid" || "$_lmms_adapter" == "video_llava" ]]; then _lmms_extra_args=",model_name=$(basename "$model_path")" fi diff --git a/oellm/scheduler.py b/oellm/scheduler.py index ace1f6bc..f09de36d 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -636,6 +636,7 @@ def _lower_suite_only(s: str) -> str: "slurm_mem": slurm_mem, "lighteval_model_args": additional_model_args, "max_num_frames": os.environ.get("MAX_NUM_FRAMES"), + "lmms_model_args": os.environ.get("LMMS_MODEL_ARGS"), "limit": limit, "venv_path": venv_path, "hf_hub_offline": _resolve_hf_hub_offline(local), diff --git a/tests/test_audio_task_groups.py b/tests/test_audio_task_groups.py index dbc34404..6683e716 100644 --- a/tests/test_audio_task_groups.py +++ b/tests/test_audio_task_groups.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest.mock import patch +import pytest import yaml from oellm.task_groups import ( @@ -282,25 +283,29 @@ def test_qwen2_audio_detected(self): assert detect_lmms_model_type("Qwen/Qwen2-Audio-7B-Instruct") == "qwen2_audio" - def test_qwen2_5_audio_detected(self): + def test_qwen2_5_omni_detected(self): from oellm.constants import detect_lmms_model_type - assert detect_lmms_model_type("Qwen/Qwen2.5-Audio-7B") == "qwen2_5_audio" + assert detect_lmms_model_type("Qwen/Qwen2.5-Omni-7B") == "qwen2_5_omni" - def test_salmonn_detected(self): + def test_video_salmonn_detected(self): from oellm.constants import detect_lmms_model_type - assert detect_lmms_model_type("tsinghua-ee/SALMONN-7B") == "salmonn" + assert detect_lmms_model_type("tsinghua-ee/video-SALMONN-2") == "video_salmonn_2" - def test_audio_flamingo_detected(self): + def test_audio_flamingo_3_detected(self): from oellm.constants import detect_lmms_model_type - assert detect_lmms_model_type("nvidia/audio-flamingo-2") == "audio_flamingo" + assert detect_lmms_model_type("nvidia/audio-flamingo-3") == "audio_flamingo_3" - def test_ultravox_detected(self): + def test_families_without_an_lmms_adapter_raise(self): + """lmms-eval has no adapter for Ultravox or Audio Flamingo 2; failing at + schedule time beats loading the wrong adapter on the node.""" from oellm.constants import detect_lmms_model_type - assert detect_lmms_model_type("fixie-ai/ultravox-v0_4") == "ultravox" + for model in ("fixie-ai/ultravox-v0_4", "nvidia/audio-flamingo-2"): + with pytest.raises(ValueError): + detect_lmms_model_type(model) def test_phi4_multimodal_detected(self): from oellm.constants import detect_lmms_model_type diff --git a/tests/test_envcheck.py b/tests/test_envcheck.py index 94508898..eee0f013 100644 --- a/tests/test_envcheck.py +++ b/tests/test_envcheck.py @@ -275,3 +275,72 @@ def test_broken_venv_path_fails(self, tmp_path): results = run_doctor_checks(venv_path=str(tmp_path / "nope")) venv_checks = [r for r in results if r.name == "venv"] assert venv_checks and venv_checks[0].status == FAIL + + +class TestLmmsAdapterPreflight: + """The resolved lmms-eval adapter must exist in the venv that runs the job.""" + + @staticmethod + def _venv(tmp_path): + (tmp_path / "venv" / "bin").mkdir(parents=True) + (tmp_path / "venv" / "bin" / "python").touch() + return str(tmp_path / "venv") + + def test_adapters_are_read_from_suite_strings(self): + from oellm.envcheck import lmms_adapters_from_suites + + suites = { + "lmms_eval:llava_hf", + " LMMS_EVAL:qwen2_vl", + "lm_eval", + "spurious_robustness:x", + } + assert lmms_adapters_from_suites(suites) == {"llava_hf", "qwen2_vl"} + + def test_unregistered_adapter_is_reported_with_close_matches( + self, tmp_path, monkeypatch + ): + import oellm.envcheck as envcheck + + monkeypatch.setattr(envcheck, "probe_import", lambda py, mod: (True, "0.7.2")) + monkeypatch.setattr( + envcheck, "lmms_registered_adapters", lambda py: {"llava_hf", "idefics2"} + ) + problems = envcheck.collect_problems( + {"lmms_eval:idefics3"}, venv_path=self._venv(tmp_path) + ) + assert len(problems) == 1 + assert "'idefics3'" in problems[0] + assert "idefics2" in problems[0] + + def test_registered_adapter_passes(self, tmp_path, monkeypatch): + import oellm.envcheck as envcheck + + monkeypatch.setattr(envcheck, "probe_import", lambda py, mod: (True, "0.7.2")) + monkeypatch.setattr(envcheck, "lmms_registered_adapters", lambda py: {"llava_hf"}) + assert ( + envcheck.collect_problems( + {"lmms_eval:llava_hf"}, venv_path=self._venv(tmp_path) + ) + == [] + ) + + def test_registry_is_not_probed_when_lmms_eval_is_missing( + self, tmp_path, monkeypatch + ): + import oellm.envcheck as envcheck + + monkeypatch.setattr( + envcheck, + "probe_import", + lambda py, mod: (False, "No module named 'lmms_eval'"), + ) + calls = [] + monkeypatch.setattr( + envcheck, "lmms_registered_adapters", lambda py: calls.append(py) or set() + ) + problems = envcheck.collect_problems( + {"lmms_eval:llava_hf"}, venv_path=self._venv(tmp_path) + ) + assert not calls + assert len(problems) == 1 and "lmms_eval" in problems[0] diff --git a/tests/test_lmms_adapters.py b/tests/test_lmms_adapters.py new file mode 100644 index 00000000..921fb790 --- /dev/null +++ b/tests/test_lmms_adapters.py @@ -0,0 +1,162 @@ +"""LMMS_MODEL_ADAPTERS must name adapters the pinned lmms-eval registers.""" + +import pytest + +from oellm.constants import LMMS_MODEL_ADAPTERS, detect_lmms_model_type + +# Registry of lmms-eval 45c766f60b6f8c153e4c72d06ca636e2db0ebcdb, the commit +# docs/VENV.md installs. Regenerate from that venv with: +# python -c "import json, lmms_eval.models as m; print(json.dumps(sorted( +# set(m.MODEL_REGISTRY_V2.list_model_names()) | set(m.AVAILABLE_SIMPLE_MODELS))))" +PINNED_LMMS_EVAL_ADAPTERS = frozenset( + [ + "aero", + "aria", + "async_hf", + "async_hf_model", + "async_openai", + "async_openai_compatible", + "async_openai_compatible_chat", + "audio_flamingo_3", + "auroracap", + "bagel", + "bagel_lmms_engine", + "bagel_umm", + "bagel_unig2u", + "baichuan_omni", + "batch_gpt4", + "cambrians", + "cambrians_vsc", + "cambrians_vsc_streaming", + "cambrians_vsr", + "claude", + "cogvlm2", + "dummy", + "dummy_video_reader", + "egogpt", + "fastvideo", + "from_log", + "fuyu", + "gemini_api", + "gemma3", + "glm4v", + "gpt4o_audio", + "gpt4v", + "huggingface", + "idefics2", + "illume_plus", + "instructblip", + "internvideo2", + "internvideo2_5", + "internvl", + "internvl2", + "internvl3", + "internvl3_5", + "internvl_hf", + "kimi_audio", + "litellm", + "litellm_chat", + "litellm_compatible", + "llama4_scout", + "llama_vid", + "llama_vision", + "llava", + "llava_hf", + "llava_onevision", + "llava_onevision1_5", + "llava_onevision2", + "llava_onevision_moviechat", + "llava_sglang", + "llava_vid", + "longva", + "longvila", + "mantis", + "minicpm_o", + "minicpm_v", + "minimonkey", + "mmada", + "moviechat", + "mplug_owl_video", + "nanovlm", + "ola", + "omnivinci", + "openai", + "openai_compatible", + "openai_compatible_chat", + "oryx", + "ovis_u1", + "penguinvl", + "phi3v", + "phi4_multimodal", + "plm", + "qwen2_5_omni", + "qwen2_5_vl", + "qwen2_audio", + "qwen2_vl", + "qwen3_5", + "qwen3_omni", + "qwen3_vl", + "qwen_image_edit", + "qwen_vl", + "qwen_vl_api", + "reka", + "ross", + "sam3", + "sglang", + "slime", + "srt_api", + "thyme", + "tinyllava", + "uni_moe_2_omni", + "videoChatGPT", + "video_llava", + "video_salmonn_2", + "videochat2", + "videochat_flash", + "videollama3", + "vila", + "vita", + "vllm", + "vllm_generate", + "vora", + "whisper", + "whisper_tt", + "whisper_vllm", + "xcomposer2_4KHD", + "xcomposer2d5", + ] +) + + +def test_every_mapped_adapter_exists_in_the_pinned_registry(): + missing = sorted( + {adapter for _, adapter in LMMS_MODEL_ADAPTERS} - PINNED_LMMS_EVAL_ADAPTERS + ) + assert not missing, ( + f"LMMS_MODEL_ADAPTERS routes to adapters the pinned lmms-eval does not " + f"register: {missing}" + ) + + +@pytest.mark.parametrize( + "model, adapter", + [ + ("HuggingFaceM4/idefics2-8b", "idefics2"), + ("llava-hf/llava-interleave-qwen-0.5b-hf", "llava_hf"), + ("Qwen/Qwen2.5-Omni-7B", "qwen2_5_omni"), + ("openai/whisper-large-v3", "whisper"), + ("moonshotai/Kimi-Audio-7B-Instruct", "kimi_audio"), + ], +) +def test_model_families_route_to_registered_adapters(model, adapter): + assert detect_lmms_model_type(model) == adapter + + +@pytest.mark.parametrize( + "model", ["HuggingFaceTB/SmolVLM-256M-Instruct", "HuggingFaceM4/Idefics3-8B-Llama3"] +) +def test_idefics3_family_is_refused_at_schedule_time(model): + """No adapter in the pinned lmms-eval can run Idefics3 checkpoints with + transformers<4.50; refusing here beats a headless model on the node.""" + with pytest.raises(ValueError): + detect_lmms_model_type(model) diff --git a/tests/test_lmms_local_args.py b/tests/test_lmms_local_args.py new file mode 100644 index 00000000..99671b5c --- /dev/null +++ b/tests/test_lmms_local_args.py @@ -0,0 +1,54 @@ +"""lmms-eval device placement is decided by the job script, not baked at schedule time.""" + +import json +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +from oellm.core.base_model_adapter import DefaultHFAdapter +from oellm.main import schedule_evals + + +def test_lmms_args_leave_device_to_the_job_script(): + args = DefaultHFAdapter().to_lmms_eval_args() + assert args.startswith("pretrained=$model_path") + assert "device_map" not in args + assert "$_lmms_extra_args" in args + + +def _render(tmp_path: Path) -> Path: + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.runner.detect_lmms_model_type", return_value="llava_hf"), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + ): + schedule_evals( + models="llava-hf/llava-interleave-qwen-0.5b-hf", + task_groups="image-realworldqa", + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + ) + return next(tmp_path.glob("**/submit_evals.sbatch")) + + +def test_job_script_resolves_lmms_device_at_run_time(tmp_path): + script_path = _render(tmp_path) + script = script_path.read_text() + assert 'case ",${LMMS_MODEL_ARGS:-}," in' in script + assert '_lmms_extra_args=",device=mps,device_map=mps"' in script + assert '_lmms_extra_args=",device_map=auto"' in script + assert '--model_args "pretrained=$model_path$_lmms_extra_args"' in script + subprocess.run(["bash", "-n", str(script_path)], check=True) + + +def test_lmms_model_args_override_is_recorded_in_provenance(tmp_path): + with patch.dict(os.environ, {"LMMS_MODEL_ARGS": "device=cpu,device_map=cpu"}): + script_path = _render(tmp_path) + provenance = json.loads((script_path.parent / "provenance.json").read_text()) + assert provenance["lmms_model_args"] == "device=cpu,device_map=cpu" + # The value is read on the node, never substituted into the script. + assert "${LMMS_MODEL_ARGS:-}" in script_path.read_text() From 9e8e64558c708103cb0770b82e666c2886b1918f Mon Sep 17 00:00:00 2001 From: islobozhan Date: Mon, 21 Sep 2026 13:27:18 +0200 Subject: [PATCH 37/44] [Contrib][Image] Add DocVQA 2026 benchmark --- docs/VENV.md | 3 + oellm/contrib/README.md | 31 ++ oellm/contrib/docvqa2026/README.md | 102 +++++ oellm/contrib/docvqa2026/__init__.py | 0 .../contrib/docvqa2026/_vendor_eval_utils.py | 171 ++++++++ oellm/contrib/docvqa2026/adapter.py | 24 ++ oellm/contrib/docvqa2026/datasets.py | 104 +++++ oellm/contrib/docvqa2026/metrics.py | 44 ++ oellm/contrib/docvqa2026/prompts.py | 7 + oellm/contrib/docvqa2026/runner.py | 125 ++++++ oellm/contrib/docvqa2026/suite.py | 150 +++++++ oellm/contrib/docvqa2026/task.py | 43 ++ pyproject.toml | 17 +- tests/test_docvqa2026.py | 389 ++++++++++++++++++ 14 files changed, 1209 insertions(+), 1 deletion(-) create mode 100644 oellm/contrib/docvqa2026/README.md create mode 100644 oellm/contrib/docvqa2026/__init__.py create mode 100644 oellm/contrib/docvqa2026/_vendor_eval_utils.py create mode 100644 oellm/contrib/docvqa2026/adapter.py create mode 100644 oellm/contrib/docvqa2026/datasets.py create mode 100644 oellm/contrib/docvqa2026/metrics.py create mode 100644 oellm/contrib/docvqa2026/prompts.py create mode 100644 oellm/contrib/docvqa2026/runner.py create mode 100644 oellm/contrib/docvqa2026/suite.py create mode 100644 oellm/contrib/docvqa2026/task.py create mode 100644 tests/test_docvqa2026.py diff --git a/docs/VENV.md b/docs/VENV.md index f55d3d39..f5324094 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -23,6 +23,9 @@ documented in `oellm/contrib//README.md`: |---|---|---| | `audio-audiobench*` | `audiobench` | [`oellm/contrib/audiobench/README.md`](../oellm/contrib/audiobench/README.md) | | `regiondial-*` | `regiondial_bench` | [`oellm/contrib/regiondial_bench/README.md`](../oellm/contrib/regiondial_bench/README.md) | +| `image-docvqa2026` | `docvqa2026` | [`oellm/contrib/docvqa2026/README.md`](../oellm/contrib/docvqa2026/README.md) | +| `spurious-robustness`, `spurious-imagenet`, `spurious-celeba` | `spurious_robustness` | [`oellm/contrib/spurious_robustness/README.md`](../oellm/contrib/spurious_robustness/README.md) | +| `spurious-urbancars` | `spurious_urbancars` | [`oellm/contrib/spurious_urbancars/README.md`](../oellm/contrib/spurious_urbancars/README.md) | Use `oellm-eval list-tasks` to see which suite a given task group routes to, and `oellm-eval doctor --venv-path --task-groups ` to verify a diff --git a/oellm/contrib/README.md b/oellm/contrib/README.md index 92e95d99..35fd5e37 100644 --- a/oellm/contrib/README.md +++ b/oellm/contrib/README.md @@ -12,6 +12,7 @@ To add your own benchmark, see the [Contributing Guide](CONTRIBUTING.md). | AudioBench | `audio-audiobench` (+ `-asr` / `-st` / `-reasoning`) | 27 judge-free audio tasks — ASR (WER), speech translation (BLEU), spoken reasoning, AudioCaps captioning — scored with AudioBench's own normalisers for paper-comparable numbers. | [arXiv:2406.16020](https://arxiv.org/abs/2406.16020) | [AudioLLMs/AudioBench](https://github.com/AudioLLMs/AudioBench) | | Spurious robustness | `spurious-robustness` (+ `-imagenet` / `-celeba`) | Zero-shot robustness to spurious correlations for OpenCLIP dual encoders: ImageNet (no spurious attribute, top-1) and CelebA (gender, 4 groups, worst-group). | [ACM MM 2026](https://github.com/gsarridis/vlm-spurious-robustness) | [gsarridis/vlm-spurious-robustness](https://github.com/gsarridis/vlm-spurious-robustness) | | UrbanCars | `spurious-urbancars` | Two-attribute spurious robustness (background + co-occurring object, 8 groups, worst-group). Separate suite so its required data directory is checked before submission. | [ACM MM 2026](https://github.com/gsarridis/vlm-spurious-robustness) | [gsarridis/vlm-spurious-robustness](https://github.com/gsarridis/vlm-spurious-robustness) | +| DocVQA 2026 | `image-docvqa2026` | Reasoning questions over multi-page documents in eight domains (business reports, comics, engineering drawings, infographics, maps, science papers, posters, slides). Scored with the competition's own strict number/unit/date matcher and ANLS fallback. | [ICDAR 2026](https://www.docvqa.org/challenges/2026) | [VLR-CVC/DocVQA2026](https://github.com/VLR-CVC/DocVQA2026) | ### RegionDial-Bench @@ -57,3 +58,33 @@ ImageNet and CelebA are staged automatically from the Hub. ImageNet is gated and UrbanCars ships as the separate `spurious_urbancars` suite because it is the only one needing a cluster-local tree: `CLUSTER_ENV_VARS` applies to every task in a suite, so bundling it would fail ImageNet and CelebA wherever no UrbanCars data exists. Split out, `URBANCARS_DATA_DIR` is validated by the login-node pre-flight before submission. It has no Hub source — see the [UrbanCars README](spurious_urbancars/README.md) for why it must be built. See the full [spurious-robustness README](spurious_robustness/README.md) for group definitions, the prompting policy, and how to read a worst-group score. + +### DocVQA 2026 + +**Metrics:** `accuracy` (primary), `macro_accuracy` (mean of the eight per-domain rates), `acc_`, plus `format_compliance` + +```bash +oellm-eval schedule \ + --models HuggingFaceTB/SmolVLM-256M-Instruct \ + --task-groups image-docvqa2026 \ + --venv-path ~/docvqa-venv +``` + +Requires a venv built with the `docvqa2026` extra. Tested with SmolVLM; other +`AutoModelForVision2Seq` checkpoints (Qwen2-VL, Idefics3) are expected to work +but have not been run. + +Only the `val` split is scorable — 80 questions over 25 documents. The test +split's answers are withheld and graded solely on the +[RRC platform](https://rrc.cvc.uab.es/?ch=34). + +The scorer's code is vendored verbatim from the competition's `eval_utils.py`, +so a prediction without the `FINAL ANSWER:` marker is wrong whatever it says, +and a numeric ground truth is matched on value *and* unit with no ANLS +fallback. `format_compliance` reports how often the model followed the output +protocol, separating a formatting failure from a reasoning one. + +Each question is asked against its whole document, about 36 pages. Models that +cannot hold that need `DOCVQA2026_MAX_PAGES`, which records the truncation and +makes scores non-comparable with the competition leaderboard. See the full +[DocVQA 2026 README](docvqa2026/README.md). diff --git a/oellm/contrib/docvqa2026/README.md b/oellm/contrib/docvqa2026/README.md new file mode 100644 index 00000000..67d10f2c --- /dev/null +++ b/oellm/contrib/docvqa2026/README.md @@ -0,0 +1,102 @@ +# DocVQA 2026 + +Reasoning questions over multi-page documents in eight domains, from the +ICDAR 2026 competition ([site](https://www.docvqa.org/challenges/2026), +[eval code](https://github.com/VLR-CVC/DocVQA2026), +[dataset](https://huggingface.co/datasets/VLR-CVC/DocVQA-2026)). + +The venv needs the suite's runtime stack: + +```bash +uv pip install -e ".[docvqa2026]" +``` + +```bash +oellm-eval schedule \ + --models HuggingFaceTB/SmolVLM-256M-Instruct \ + --task-groups image-docvqa2026 \ + --venv-path /path/to/.venv +``` + +**Metrics:** `accuracy` (primary, over all questions), `macro_accuracy` (mean +of the eight per-domain rates), and `acc_` for each domain. + +Both averages are reported because they answer different questions. On `val` +they are equal — each of the eight domains holds exactly 10 of the 80 +questions — so the competition's per-domain table reproduces either way. They +would diverge on a split with uneven domains. + +## Only `val` is scorable + +`val` is 25 documents and 80 questions. The `test` split ships with its +answers withheld and is graded solely on the +[RRC platform](https://rrc.cvc.uab.es/?ch=34). + +## Scoring is the competition's own code + +The body of `_vendor_eval_utils.py` (everything below its provenance +docstring) is the official `eval_utils.py`, verbatim, and the file is +excluded from ruff so no formatter can drift it. To verify against a clone: + +```bash +diff <(tail -n +7 oellm/contrib/docvqa2026/_vendor_eval_utils.py) /path/to/DocVQA2026/eval_utils.py +``` + +`tests/test_docvqa2026.py` pins the branches that decide scores. + +Three behaviours of that scorer decide most scores, and none of them are +bugs to fix here: + +- A prediction without the `FINAL ANSWER:` marker is wrong whatever it says, + so the prompt in `prompts.py` is the competition's, verbatim. +- When the ground truth parses as a number, a failed strict match returns + wrong **without** the ANLS fallback. 37 of the 80 val answers parse as + numbers, so nearly half the benchmark is exact number-and-unit equality. +- Values must match *and* units must match; `50 g` scores zero against + `50 kg`. + +## Pages, and why your number may not be comparable + +Each question is asked against its whole document — 905 page images across 25 +documents, about 36 pages each, roughly 50k image tokens per question. Models +that cannot hold that need `DOCVQA2026_MAX_PAGES`: + +```bash +DOCVQA2026_MAX_PAGES=4 oellm-eval schedule --task-groups image-docvqa2026 ... +``` + +The variable is read from the job's environment at run time, so when +submitting a previously generated script by hand, set it in that shell too: +`DOCVQA2026_MAX_PAGES=4 sbatch .../submit_evals.sbatch`. + +A capped run records `max_pages` and `n_truncated_documents` in its results +and logs a warning. **Capped scores are not comparable with the competition +leaderboard**, which grades the full document. The published baselines +(Gemini 3 Pro 0.375, GPT-5.2 0.350) are frontier API models reading every +page; a small local VLM on a handful of pages is measuring something else. + +## Decoding + +Generation is greedy (`do_sample=False`) with up to 2048 new tokens; the +baselines were sampled at temperature 1.0, so scores are reproducible here +but not sampled the same way. The prompt asks for step-by-step reasoning +*before* `FINAL ANSWER:`, so a model that is cut off by the token cap loses +its marker and scores wrong for a formatting failure it did not commit. +`n_hit_token_limit` in the results counts those answers; raise the cap for +verbose reasoners: + +```bash +DOCVQA2026_MAX_NEW_TOKENS=4096 oellm-eval schedule --task-groups image-docvqa2026 ... +``` + +Every prediction is also appended, with the raw model text, to a +`.partial.jsonl` next to the results JSON as it is scored, so a run +that dies late keeps what it produced. The collector ignores that file. + +## Models + +Loading goes through `AutoModelForVision2Seq` + `AutoProcessor` with a +chat-template message carrying one image slot per page, so no per-family +code exists. Tested end to end with `HuggingFaceTB/SmolVLM-256M-Instruct`. +Qwen2-VL and Idefics3 are in the same auto-class mapping and expected to +work, but have not been run here. diff --git a/oellm/contrib/docvqa2026/__init__.py b/oellm/contrib/docvqa2026/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oellm/contrib/docvqa2026/_vendor_eval_utils.py b/oellm/contrib/docvqa2026/_vendor_eval_utils.py new file mode 100644 index 00000000..66536837 --- /dev/null +++ b/oellm/contrib/docvqa2026/_vendor_eval_utils.py @@ -0,0 +1,171 @@ +"""Official DocVQA 2026 scorer and prompt, copied verbatim below this docstring +from https://github.com/VLR-CVC/DocVQA2026 (eval_utils.py). Do not edit or +reformat: any change alters competition scores. Excluded from ruff in +pyproject.toml. +""" + +import Levenshtein +import re +import string +import ast +from dateutil import parser +from datetime import date + +# --- 1. ANLS HELPER FUNCTIONS --- +def get_anls(s1, s2): + s1 = s1.lower().strip() + s2 = s2.lower().strip() + if not s1 and not s2: return 1.0 + if not s1 or not s2: return 0.0 + dist = Levenshtein.distance(s1, s2) + max_len = max(len(s1), len(s2)) + return 1.0 - (dist / max_len) + +def is_string_correct(prediction, ground_truths, threshold=0.80): + max_score = 0.0 + for gt in ground_truths: + score = get_anls(prediction, gt) + if score > max_score: + max_score = score + return max_score >= threshold + +# --- 2. UNIT PARSING LOGIC --- +def parse_magnitude_unit(text): + """ + Splits a string into a numeric float and a unit string. + """ + text = text.lower().strip() + + match = re.match(r'^(-?\d+(?:\.\d+)?)\s*(.*)$', text) # Check reaction to sections (S.O.) + + if not match: + return None, None + + number_str = match.group(1) + unit_str = match.group(2).strip() + + try: + val = float(number_str) + return val, unit_str + except ValueError: + return None, None + +# --- 3. MAIN EVALUATION --- +def evaluate_docvqa_prediction(raw_prediction, ground_truth): + + if not isinstance(raw_prediction, str): + raw_prediction = str(raw_prediction) + + marker = "FINAL ANSWER:" + if marker not in raw_prediction: + return False, raw_prediction + + extracted_answer = raw_prediction.split(marker)[-1].strip() + + gt_candidates = [] + try: + parsed_gt = ast.literal_eval(str(ground_truth)) + if isinstance(parsed_gt, list): + gt_candidates = [str(x) for x in parsed_gt] + else: + gt_candidates = [str(ground_truth)] + except (ValueError, SyntaxError): + gt_candidates = [str(ground_truth)] + + # --- A. STRICT MATCHING LOGIC --- + def check_strict_match(pred_text, gt_text): + """ + Returns True ONLY if: + 1. Both are valid numbers. + 2. The numeric values are equal. + 3. The units are identical. + """ + pred_val, pred_unit = parse_magnitude_unit(pred_text) + gt_val, gt_unit = parse_magnitude_unit(gt_text) + + # Check Number + Unit Match + if pred_val is not None and gt_val is not None: + if pred_val == gt_val: + if pred_unit == gt_unit: + return True + return False + + # Check Date Match + try: + p_clean = pred_text.strip() + g_clean = gt_text.strip() + version_regex = r'^\d+\.\d+\.\d+$' + if re.match(version_regex, p_clean) or re.match(version_regex, g_clean): + return p_clean == g_clean + + if len(p_clean) >= 6 and len(g_clean) >= 6: # Is >= 6 necessary? If yes is not >=6 12.0.0 has 6 characters (S.O.) + pred_date = parser.parse(p_clean, fuzzy=False).date() + gt_date = parser.parse(g_clean, fuzzy=False).date() + return pred_date == gt_date + + except (ValueError, TypeError, OverflowError): + pass + + return False + + # --- B. EXECUTION --- + # 1. Try Strict Match & Detect Numeric GT + + # ----- This section is unnecesary (S.O.) You can just check the first element + gt_is_numeric = False + + for gt in gt_candidates: + if check_strict_match(extracted_answer, gt): + return True, extracted_answer + + gt_val, gt_unit = parse_magnitude_unit(gt) + if gt_val is not None: + gt_is_numeric = True + + if gt_is_numeric: + return False, extracted_answer + #----- + + # 3. RELAXED TEXT MATCH (ANLS) + translator = str.maketrans(string.punctuation, ' ' * len(string.punctuation)) + + def clean_text(text): + t = text.lower().translate(translator) + t = re.sub(r'\b(a|an|the)\b', ' ', t) + return " ".join(t.split()) + + clean_pred = clean_text(extracted_answer) + clean_gt_candidates = [clean_text(gt) for gt in gt_candidates] + + is_correct = is_string_correct(clean_pred, clean_gt_candidates, threshold=0.9) + + return is_correct, extracted_answer + +def get_evaluation_prompt() -> str: + MASTER_PROMPT = ( + "ACT AS an expert Document Visual Question Answering (DocVQA) system. " + "ANALYZE the provided images to extract precise information.\n\n" + "### MANDATORY RESPONSE RULES:\n" + "1. SOURCE ADHERENCE: If the question is unanswerable from the document, respond ONLY with \"Unknown\".\n" + "2. LIST FORMATTING: List multiple answers in order of appearance, separated by a comma and a single space (e.g., \"Answer A, Answer B\"). Do NOT use \"and\".\n" + "3. NUMBERS & UNITS:\n" + " - Convert units to their standardized abbreviation (e.g., use \"kg\" not \"kilograms\", \"m\" not \"meters\").\n" + " - Place a single space between the number and the unit (e.g., \"50 kg\", \"10 USD\").\n" + "4. PERCENTAGES: For percentages, attach the '%' symbol directly to the number with NO space (e.g., \"50%\", not \"50 %\").\n" + "5. DATE FORMATTING: Convert all dates to YYYY-MM-DD format (e.g., convert \"Jan 1st 24\" to \"2024-01-01\").\n" + "6. DECIMAL FORMATTING: Decimals should be separated by a single period (e.g., \"3.14\", not \"3,14\").\n" + "7. THOUSANDS SEPARATOR: Do NOT use commas as thousands separators (e.g., \"1000\", not \"1,000\").\n" + "8. NO FILLER: Output ONLY the result. Do not frame with sentences like \"The answer is...\"." + + "\n\n### REASONING PROTOCOL:\n" + "1. Perform exhaustive step-by-step reasoning to locate and verify the data.\n" + "2. Verify if the data contains a date, number, or unit.\n" + "3. Step-by-step, transform the data to match the MANDATORY RESPONSE RULES (e.g., converting date format).\n" + + "\n\n### OUTPUT FORMAT:\n" + "After your analysis, you MUST provide the final result in the following format:\n" + "FINAL ANSWER: [Your exact formatted answer]\n" + "Ensure the content inside [FINAL ANSWER] strictly follows the MANDATORY RESPONSE RULES." + ) + + return MASTER_PROMPT diff --git a/oellm/contrib/docvqa2026/adapter.py b/oellm/contrib/docvqa2026/adapter.py new file mode 100644 index 00000000..e0baf49c --- /dev/null +++ b/oellm/contrib/docvqa2026/adapter.py @@ -0,0 +1,24 @@ +"""Model adapter for DocVQA 2026.""" + +from __future__ import annotations + +from oellm.core.base_model_adapter import BaseModelAdapter + + +class DocVQA2026Adapter(BaseModelAdapter): + def __init__(self, model_path: str) -> None: + self._model_path = str(model_path) + + @property + def model_path(self) -> str: + return self._model_path + + def to_lm_eval_args(self) -> str: + return f"pretrained={self._model_path}" + + def to_lmms_eval_args(self) -> str: + return f"pretrained={self._model_path}" + + def to_contrib_flags(self) -> str | None: + """No flags: one transformers path serves every supported checkpoint.""" + return None diff --git a/oellm/contrib/docvqa2026/datasets.py b/oellm/contrib/docvqa2026/datasets.py new file mode 100644 index 00000000..5a1e56a7 --- /dev/null +++ b/oellm/contrib/docvqa2026/datasets.py @@ -0,0 +1,104 @@ +"""Loading the DocVQA 2026 val split as one sample per question.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +HF_REPO = "VLR-CVC/DocVQA-2026" +SPLIT = "val" + + +@dataclass +class Sample: + """One scorable question with the still-encoded pages of its document.""" + + question_id: str + doc_id: str + doc_category: str + question: str + answer: str + encoded_pages: list = field(repr=False, default_factory=list) + n_pages_total: int = 0 + + @property + def pages_truncated(self) -> bool: + return len(self.encoded_pages) < self.n_pages_total + + +def decode_pages(sample: Sample) -> list: + """Decode one sample's kept pages to PIL RGB images.""" + return [_decode_page(page) for page in sample.encoded_pages] + + +def _decode_page(entry: dict): + import io + + from PIL import Image + + Image.MAX_IMAGE_PIXELS = None + if entry.get("bytes") is not None: + return Image.open(io.BytesIO(entry["bytes"])).convert("RGB") + return Image.open(entry["path"]).convert("RGB") + + +def read_max_pages(env: dict[str, str]) -> int | None: + raw = env.get("DOCVQA2026_MAX_PAGES", "").strip() + if not raw: + return None + try: + value = int(raw) + except ValueError: + raise ValueError( + f"DOCVQA2026_MAX_PAGES must be an integer >= 1, got {raw!r}" + ) from None + if value < 1: + raise ValueError(f"DOCVQA2026_MAX_PAGES must be >= 1, got {value!r}") + return value + + +def load_val(limit: int | None = None, max_pages: int | None = None) -> list[Sample]: + """Return the val split's questions in dataset order, read via pyarrow.""" + import pyarrow.parquet as pq + from huggingface_hub import hf_hub_download + + path = hf_hub_download(HF_REPO, f"{SPLIT}.parquet", repo_type="dataset") + columns = ["doc_id", "doc_category", "questions", "answers", "document"] + + samples: list[Sample] = [] + parquet = pq.ParquetFile(path) + for batch in parquet.iter_batches(batch_size=1, columns=columns): + for row in batch.to_pylist(): + encoded = list(row["document"] or []) + kept = encoded[:max_pages] if max_pages else encoded + answers = dict( + zip( + row["answers"]["question_id"], + row["answers"]["answer"], + strict=True, + ) + ) + for qid, question in zip( + row["questions"]["question_id"], + row["questions"]["question"], + strict=True, + ): + if qid not in answers: + logger.warning("question %s has no answer; skipping", qid) + continue + samples.append( + Sample( + question_id=qid, + doc_id=row["doc_id"], + doc_category=row["doc_category"], + question=question, + answer=answers[qid], + encoded_pages=kept, + n_pages_total=len(encoded), + ) + ) + if limit and len(samples) >= limit: + return samples + return samples diff --git a/oellm/contrib/docvqa2026/metrics.py b/oellm/contrib/docvqa2026/metrics.py new file mode 100644 index 00000000..56942c8c --- /dev/null +++ b/oellm/contrib/docvqa2026/metrics.py @@ -0,0 +1,44 @@ +"""Scoring and aggregation for DocVQA 2026.""" + +from __future__ import annotations + +from collections import defaultdict + +MARKER = "FINAL ANSWER:" + + +def score_prediction(raw_prediction: str, ground_truth: str) -> tuple[bool, str, bool]: + """Official verdict for one prediction: (correct, extracted, has_marker).""" + from oellm.contrib.docvqa2026._vendor_eval_utils import ( + evaluate_docvqa_prediction, + ) + + correct, extracted = evaluate_docvqa_prediction(raw_prediction, ground_truth) + return bool(correct), extracted, MARKER in str(raw_prediction) + + +def aggregate(records: list[dict]) -> dict[str, float | int]: + """Aggregate per-question verdicts into the reported metric set.""" + if not records: + raise RuntimeError("DocVQA 2026 evaluation produced no samples") + + per_category: dict[str, list[bool]] = defaultdict(list) + for r in records: + per_category[r["doc_category"]].append(bool(r["correct"])) + + n_correct = sum(bool(r["correct"]) for r in records) + category_rates = {cat: sum(v) / len(v) for cat, v in sorted(per_category.items())} + + n_marked = sum(bool(r.get("has_marker")) for r in records) + + metrics: dict[str, float | int] = { + "accuracy": n_correct / len(records), + "format_compliance": n_marked / len(records), + "macro_accuracy": sum(category_rates.values()) / len(category_rates), + "n_questions": len(records), + "n_correct": n_correct, + "n_categories": len(category_rates), + } + for cat, rate in category_rates.items(): + metrics[f"acc_{cat}"] = rate + return metrics diff --git a/oellm/contrib/docvqa2026/prompts.py b/oellm/contrib/docvqa2026/prompts.py new file mode 100644 index 00000000..a107880a --- /dev/null +++ b/oellm/contrib/docvqa2026/prompts.py @@ -0,0 +1,7 @@ +"""The competition's baseline prompt, taken from the official scorer.""" + +from oellm.contrib.docvqa2026._vendor_eval_utils import get_evaluation_prompt + +MASTER_PROMPT = get_evaluation_prompt() + +__all__ = ["MASTER_PROMPT", "get_evaluation_prompt"] diff --git a/oellm/contrib/docvqa2026/runner.py b/oellm/contrib/docvqa2026/runner.py new file mode 100644 index 00000000..2194705a --- /dev/null +++ b/oellm/contrib/docvqa2026/runner.py @@ -0,0 +1,125 @@ +"""Generation for DocVQA 2026: one prompt per question, all pages attached.""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +DEFAULT_MAX_NEW_TOKENS = 2048 + + +def resolve_device() -> str: + import torch + + if torch.cuda.is_available(): + return "cuda" + if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available(): + return "mps" + return "cpu" + + +def read_limit(env: dict[str, str]) -> int | None: + raw = env.get("LIMIT", "").strip() + if not raw: + return None + try: + value = int(raw) + except ValueError: + raise ValueError(f"LIMIT must be an integer, got {raw!r}") from None + return value if value > 0 else None + + +def read_max_new_tokens(env: dict[str, str]) -> int: + raw = env.get("DOCVQA2026_MAX_NEW_TOKENS", "").strip() + if not raw: + return DEFAULT_MAX_NEW_TOKENS + try: + value = int(raw) + except ValueError: + raise ValueError( + f"DOCVQA2026_MAX_NEW_TOKENS must be an integer >= 1, got {raw!r}" + ) from None + if value < 1: + raise ValueError(f"DOCVQA2026_MAX_NEW_TOKENS must be >= 1, got {value!r}") + return value + + +def load_model(model_path: str, device: str): + import torch + from transformers import AutoModelForVision2Seq, AutoProcessor + + logger.info("Loading vision-language model %s on %s", model_path, device) + processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True) + dtype = {"cuda": torch.bfloat16, "mps": torch.float16}.get(device, torch.float32) + model = AutoModelForVision2Seq.from_pretrained( + model_path, + torch_dtype=dtype, + trust_remote_code=True, + ).to(device) + model.eval() + return model, processor + + +def build_messages(question: str, n_images: int, prompt: str) -> list[dict]: + content = [{"type": "image"} for _ in range(n_images)] + content.append({"type": "text", "text": f"{prompt}\n\nQUESTION: {question}"}) + return [{"role": "user", "content": content}] + + +def generate_answer( + model, + processor, + question: str, + images: list, + prompt: str, + device: str, + max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS, +) -> tuple[str, bool]: + """Raw model output for one question and whether the token cap cut it off.""" + import torch + + messages = build_messages(question, len(images), prompt) + text = processor.apply_chat_template(messages, add_generation_prompt=True) + inputs = processor(text=text, images=images or None, return_tensors="pt").to(device) + + with torch.no_grad(): + generated = model.generate( + **inputs, max_new_tokens=max_new_tokens, do_sample=False + ) + prompt_len = inputs["input_ids"].shape[1] + new_tokens = generated[0][prompt_len:] + hit_limit = new_tokens.shape[0] >= max_new_tokens + return processor.decode(new_tokens, skip_special_tokens=True), hit_limit + + +def write_results( + output_path: Path, model_path: str, task: str, n_shot: int, metrics: dict +) -> None: + result_json = { + "model_name_or_path": model_path, + "results": {task: metrics}, + "configs": {task: {"num_fewshot": n_shot}}, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + json.dump(result_json, f, indent=2) + logger.info("Results written to %s", output_path) + + +def parse_suite_results( + data: dict, task_prefix: str +) -> tuple[str, str, int, dict[str, float]] | None: + """Claim *data* if it carries this suite's task and metric shape.""" + results = data.get("results", {}) + for task_name, task_results in results.items(): + if not task_name.startswith(task_prefix) or not isinstance(task_results, dict): + continue + if "macro_accuracy" not in task_results: + continue + model_id = data.get("model_name_or_path") or data.get("model_name", "unknown") + n_shot = data.get("configs", {}).get(task_name, {}).get("num_fewshot", 0) + return (model_id, task_name, int(n_shot), dict(task_results)) + return None diff --git a/oellm/contrib/docvqa2026/suite.py b/oellm/contrib/docvqa2026/suite.py new file mode 100644 index 00000000..4437c8fc --- /dev/null +++ b/oellm/contrib/docvqa2026/suite.py @@ -0,0 +1,150 @@ +"""DocVQA 2026 contrib suite (https://github.com/VLR-CVC/DocVQA2026).""" + +from __future__ import annotations + +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + +SUITE_NAME = "docvqa2026" + +CLUSTER_ENV_VARS: list[str] = [] + +from oellm.contrib.docvqa2026.task import DocVQA2026ValTask # noqa: E402 + +_TASK = DocVQA2026ValTask() + +TASK_GROUPS: dict = DocVQA2026ValTask.to_task_groups_dict() + + +def detect_model_flags(model_path: str) -> str | None: + """Delegate to DocVQA2026Adapter.to_contrib_flags().""" + from oellm.contrib.docvqa2026.adapter import DocVQA2026Adapter + + return DocVQA2026Adapter(model_path).to_contrib_flags() + + +def run( + *, + model_path: str, + task: str, + n_shot: int, + output_path: Path, + model_flags: str | None, + env: dict[str, str], +) -> None: + """Evaluate *task* and write a lmms-eval-compatible JSON to *output_path*.""" + if task != _TASK.engine_task_name: + raise ValueError(f"Unknown task {task!r}. Expected {_TASK.engine_task_name!r}") + + import json + + from oellm.contrib.docvqa2026.datasets import ( + decode_pages, + load_val, + read_max_pages, + ) + from oellm.contrib.docvqa2026.metrics import aggregate, score_prediction + from oellm.contrib.docvqa2026.prompts import MASTER_PROMPT + from oellm.contrib.docvqa2026.runner import ( + generate_answer, + load_model, + read_limit, + read_max_new_tokens, + resolve_device, + write_results, + ) + + limit = read_limit(env) + max_pages = read_max_pages(env) + max_new_tokens = read_max_new_tokens(env) + + device = resolve_device() + model, processor = load_model(model_path, device) + + samples = load_val(limit=limit, max_pages=max_pages) + logger.info( + "DocVQA 2026: %d questions over %d documents", + len(samples), + len({s.doc_id for s in samples}), + ) + + partial_path = output_path.parent / (output_path.stem + ".partial.jsonl") + partial_path.parent.mkdir(parents=True, exist_ok=True) + + records = [] + truncated_docs: set[str] = set() + hit_token_limit = 0 + current_doc = None + images: list = [] + with open(partial_path, "w") as partial: + for i, sample in enumerate(samples, 1): + if sample.doc_id != current_doc: + images = decode_pages(sample) + current_doc = sample.doc_id + raw, hit_limit = generate_answer( + model, + processor, + sample.question, + images, + MASTER_PROMPT, + device, + max_new_tokens=max_new_tokens, + ) + correct, extracted, has_marker = score_prediction(raw, sample.answer) + if sample.pages_truncated: + truncated_docs.add(sample.doc_id) + hit_token_limit += hit_limit + record = { + "question_id": sample.question_id, + "doc_category": sample.doc_category, + "correct": correct, + "extracted": extracted, + "has_marker": has_marker, + "hit_token_limit": hit_limit, + "raw": raw, + } + records.append(record) + partial.write(json.dumps(record, ensure_ascii=False) + "\n") + partial.flush() + if i % 10 == 0 or i == len(samples): + logger.info("scored %d/%d questions", i, len(samples)) + + metrics = aggregate(records) + metrics["max_pages"] = max_pages if max_pages else 0 + metrics["n_truncated_documents"] = len(truncated_docs) + metrics["max_new_tokens"] = max_new_tokens + metrics["n_hit_token_limit"] = hit_token_limit + if hit_token_limit: + logger.warning( + "%d/%d answers were cut off at DOCVQA2026_MAX_NEW_TOKENS=%d before " + "finishing; a cut-off answer has no marker and scores wrong", + hit_token_limit, + len(records), + max_new_tokens, + ) + if not metrics["format_compliance"]: + logger.warning( + "no prediction carried the %r marker, so every answer scores wrong " + "regardless of content — this measures instruction following, not " + "document reasoning", + "FINAL ANSWER:", + ) + if truncated_docs: + logger.warning( + "%d/%d documents were truncated (DOCVQA2026_MAX_PAGES=%s); " + "scores are not comparable with the competition leaderboard", + len(truncated_docs), + len({s.doc_id for s in samples}), + max_pages, + ) + + write_results(output_path, model_path, task, n_shot, metrics) + + +def parse_results(data: dict) -> tuple[str, str, int, dict[str, float]] | None: + """Claim *data* if it is this suite's output, else return None.""" + from oellm.contrib.docvqa2026.runner import parse_suite_results + + return parse_suite_results(data, "docvqa2026") diff --git a/oellm/contrib/docvqa2026/task.py b/oellm/contrib/docvqa2026/task.py new file mode 100644 index 00000000..a0b6bad3 --- /dev/null +++ b/oellm/contrib/docvqa2026/task.py @@ -0,0 +1,43 @@ +"""Task definition for the DocVQA 2026 competition benchmark.""" + +from __future__ import annotations + +from oellm.core.base_task import BaseTask + +SUITE_NAME = "docvqa2026" + + +class DocVQA2026ValTask(BaseTask): + """DocVQA 2026 validation split: 80 questions over 25 multi-page documents.""" + + @property + def name(self) -> str: + return "docvqa2026_val" + + @property + def suite(self) -> str: + return SUITE_NAME + + @property + def task_group_name(self) -> str: + return "image-docvqa2026" + + @property + def n_shots(self) -> list[int]: + return [0] + + @property + def primary_metric(self) -> str: + return "accuracy" + + @property + def description(self) -> str: + return ( + "DocVQA 2026 (ICDAR) reasoning over multi-page documents in eight " + "domains, scored with the competition's own strict number/unit/date " + "matcher and ANLS fallback." + ) + + @property + def hf_dataset_files(self) -> list[dict]: + return [{"repo_id": "VLR-CVC/DocVQA-2026", "patterns": ["val.parquet"]}] diff --git a/pyproject.toml b/pyproject.toml index e0abc203..80afe81c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dev = [ "pytest>=8.4.1", "pytest-timeout>=2.3.1", "pre-commit", + "Levenshtein", ] # Text-evaluation engine base (lm-eval-harness + dependencies). Pair with # ``image`` / ``audio`` for a "general" venv that runs text + multimodal @@ -46,6 +47,13 @@ image = [ "decord; platform_system != 'Darwin'", "eva-decord; platform_system == 'Darwin' and python_version < '3.12'", ] +docvqa2026 = [ + "Levenshtein", + "python-dateutil", + "torch", + "transformers>=4.45,<4.50", + "pillow", +] video = [ "transformers>=4.45,<4.50", "backoff", @@ -148,6 +156,10 @@ conflicts = [ { extra = "video" }, { extra = "evalchemy" }, ], + [ + { extra = "docvqa2026" }, + { extra = "evalchemy" }, + ], ] [tool.ruff] @@ -155,7 +167,10 @@ line-length = 90 target-version = "py312" # Vendored verbatim from QwenLM/PolyMath (eval/scripts.py); keep it byte-for-byte # rather than reflowing/relinting upstream code. -extend-exclude = ["oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py"] +extend-exclude = [ + "oellm/resources/custom_lm_eval_tasks/polymath/polymath_eval.py", + "oellm/contrib/docvqa2026/_vendor_eval_utils.py", +] [tool.ruff.lint] select = [ diff --git a/tests/test_docvqa2026.py b/tests/test_docvqa2026.py new file mode 100644 index 00000000..337abec5 --- /dev/null +++ b/tests/test_docvqa2026.py @@ -0,0 +1,389 @@ +"""Tests for the DocVQA 2026 contrib suite.""" + +import pytest + + +@pytest.fixture(scope="module") +def scorer(): + from oellm.contrib.docvqa2026 import _vendor_eval_utils + + return _vendor_eval_utils + + +class TestSuiteWiring: + def test_staging_flows_through_the_scheduler_path(self): + from oellm.task_groups import _collect_dataset_specs, _collect_hf_dataset_files + + assert _collect_dataset_specs(["image-docvqa2026"]) == [] + assert _collect_hf_dataset_files(["image-docvqa2026"]) == [ + {"repo_id": "VLR-CVC/DocVQA-2026", "patterns": ["val.parquet"]} + ] + + def test_group_and_metric_registered(self): + from oellm.results import _load_task_metrics + from oellm.task_groups import _expand_task_groups + + assert _load_task_metrics()["docvqa2026_val"] == "accuracy" + assert [ + (r.task, r.n_shot, r.suite) for r in _expand_task_groups(["image-docvqa2026"]) + ] == [("docvqa2026_val", 0, "docvqa2026")] + + def test_group_name_follows_the_core_docvqa_pairing(self): + """image-docvqa/docvqa_val already exists; 2026 must not collide.""" + from oellm.task_groups import _expand_task_groups + + core = _expand_task_groups(["image-docvqa"]) + ours = _expand_task_groups(["image-docvqa2026"]) + assert [r.task for r in core] == ["docvqa_val"] + assert [r.task for r in ours] == ["docvqa2026_val"] + assert {r.suite for r in core} != {r.suite for r in ours} + + def test_unknown_task_is_rejected(self, tmp_path): + from oellm.contrib.docvqa2026 import suite + + with pytest.raises(ValueError, match="Unknown task"): + suite.run( + model_path="m", + task="docvqa_val", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags=None, + env={}, + ) + + def test_parse_results_claims_own_and_rejects_foreign(self): + from oellm.contrib.docvqa2026 import suite + + mine = { + "model_name_or_path": "HuggingFaceTB/SmolVLM-256M-Instruct", + "results": {"docvqa2026_val": {"accuracy": 0.05, "macro_accuracy": 0.05}}, + "configs": {"docvqa2026_val": {"num_fewshot": 0}}, + } + assert suite.parse_results(mine) == ( + "HuggingFaceTB/SmolVLM-256M-Instruct", + "docvqa2026_val", + 0, + {"accuracy": 0.05, "macro_accuracy": 0.05}, + ) + assert suite.parse_results({"results": {"docvqa_val": {"anls": 0.7}}}) is None + + +class TestAggregation: + def test_reports_both_averages_and_per_category(self): + from oellm.contrib.docvqa2026.metrics import aggregate + + records = [ + {"correct": True, "doc_category": "maps"}, + {"correct": False, "doc_category": "maps"}, + {"correct": False, "doc_category": "maps"}, + {"correct": True, "doc_category": "slide"}, + {"correct": True, "doc_category": "slide"}, + ] + m = aggregate(records) + assert m["accuracy"] == pytest.approx(0.6) + assert m["macro_accuracy"] == pytest.approx(2 / 3) + assert m["acc_maps"] == pytest.approx(1 / 3) + assert m["acc_slide"] == 1.0 + assert m["n_questions"] == 5 and m["n_correct"] == 3 + + def test_averages_agree_when_categories_are_balanced(self): + """The val split has 10 questions per category, so both must coincide.""" + from oellm.contrib.docvqa2026.metrics import aggregate + + records = [ + {"correct": i < 4, "doc_category": c} + for c in ("maps", "slide") + for i in range(10) + ] + m = aggregate(records) + assert m["accuracy"] == pytest.approx(m["macro_accuracy"]) + + def test_empty_run_is_an_error_not_a_zero(self): + from oellm.contrib.docvqa2026.metrics import aggregate + + with pytest.raises(RuntimeError, match="no samples"): + aggregate([]) + + +class TestPageCap: + @pytest.mark.parametrize("raw,expected", [("", None), ("4", 4), (" 8 ", 8)]) + def test_read_max_pages(self, raw, expected): + from oellm.contrib.docvqa2026.datasets import read_max_pages + + assert read_max_pages({"DOCVQA2026_MAX_PAGES": raw}) == expected + + def test_zero_is_rejected(self): + from oellm.contrib.docvqa2026.datasets import read_max_pages + + with pytest.raises(ValueError, match=">= 1"): + read_max_pages({"DOCVQA2026_MAX_PAGES": "0"}) + + def test_truncation_is_visible_on_the_sample(self): + from oellm.contrib.docvqa2026.datasets import Sample + + s = Sample("q1", "d1", "maps", "q?", "a", encoded_pages=[1, 2], n_pages_total=36) + assert s.pages_truncated is True + assert Sample("q1", "d1", "maps", "q?", "a", [1], 1).pages_truncated is False + + +class TestGenerationSettings: + @pytest.mark.parametrize("raw,expected", [("", 2048), ("512", 512), (" 4096 ", 4096)]) + def test_read_max_new_tokens(self, raw, expected): + from oellm.contrib.docvqa2026.runner import read_max_new_tokens + + assert read_max_new_tokens({"DOCVQA2026_MAX_NEW_TOKENS": raw}) == expected + + @pytest.mark.parametrize("raw", ["0", "-1", "many"]) + def test_bad_max_new_tokens_names_the_variable(self, raw): + from oellm.contrib.docvqa2026.runner import read_max_new_tokens + + with pytest.raises(ValueError, match="DOCVQA2026_MAX_NEW_TOKENS"): + read_max_new_tokens({"DOCVQA2026_MAX_NEW_TOKENS": raw}) + + def test_no_pages_means_no_image_slots(self): + from oellm.contrib.docvqa2026.runner import build_messages + + content = build_messages("q?", 0, "PROMPT")[0]["content"] + assert [c["type"] for c in content] == ["text"] + assert [c["type"] for c in build_messages("q?", 3, "P")[0]["content"]] == [ + "image", + "image", + "image", + "text", + ] + + +class TestPrompt: + def test_prompt_demands_the_marker_the_scorer_requires(self): + from oellm.contrib.docvqa2026.prompts import MASTER_PROMPT + + assert "FINAL ANSWER:" in MASTER_PROMPT + + +class TestRunEndToEnd: + """run() orchestration with the dataset and model stubbed out.""" + + @pytest.fixture + def stub_run(self, monkeypatch): + from oellm.contrib.docvqa2026 import datasets as ds_mod + from oellm.contrib.docvqa2026 import runner as run_mod + from oellm.contrib.docvqa2026 import suite + + samples = [ + ds_mod.Sample("q1", "d1", "maps", "how many?", "4", ["enc"], 36), + ds_mod.Sample("q2", "d1", "maps", "which town?", "Wareham", ["enc"], 36), + ds_mod.Sample("q3", "d2", "slide", "what colour?", "green", ["enc"], 1), + ] + replies = { + "how many?": "FINAL ANSWER: 4", + "which town?": "FINAL ANSWER: Boston", + "what colour?": "the colour is green", + } + monkeypatch.setattr( + ds_mod, "load_val", lambda limit=None, max_pages=None: samples + ) + monkeypatch.setattr(ds_mod, "decode_pages", lambda sample: ["img"]) + monkeypatch.setattr(run_mod, "load_model", lambda *a, **k: ("model", "proc")) + monkeypatch.setattr(run_mod, "resolve_device", lambda: "cpu") + monkeypatch.setattr( + run_mod, + "generate_answer", + lambda model, processor, question, images, prompt, device, max_new_tokens: ( + replies[question], + False, + ), + ) + return suite + + def test_writes_scored_results(self, stub_run, tmp_path): + out = tmp_path / "results" / "docvqa.json" + stub_run.run( + model_path="stub/vlm", + task="docvqa2026_val", + n_shot=0, + output_path=out, + model_flags=None, + env={"DOCVQA2026_MAX_PAGES": "1"}, + ) + import json + + data = json.loads(out.read_text()) + metrics = data["results"]["docvqa2026_val"] + assert metrics["n_questions"] == 3 + assert metrics["n_correct"] == 1 + assert metrics["accuracy"] == pytest.approx(1 / 3) + assert metrics["macro_accuracy"] == pytest.approx(0.25) + assert metrics["acc_maps"] == 0.5 and metrics["acc_slide"] == 0.0 + assert metrics["max_pages"] == 1 + assert metrics["format_compliance"] == pytest.approx(2 / 3) + assert metrics["n_truncated_documents"] == 1 + assert metrics["max_new_tokens"] == 2048 + assert metrics["n_hit_token_limit"] == 0 + + def test_results_round_trip_through_the_collector(self, stub_run, tmp_path): + import csv + + from oellm.results import collect_results + + run_dir = tmp_path / "run" + stub_run.run( + model_path="stub/vlm", + task="docvqa2026_val", + n_shot=0, + output_path=run_dir / "results" / "docvqa.json", + model_flags=None, + env={}, + ) + out_csv = run_dir / "eval.csv" + collect_results(str(run_dir), str(out_csv)) + rows = list(csv.DictReader(open(out_csv))) + assert [(r["task"], r["metric_name"]) for r in rows] == [ + ("docvqa2026_val", "accuracy") + ] + assert float(rows[0]["performance"]) == pytest.approx(1 / 3) + + +def _cached_val_parquet(): + """Path to the val parquet if already in the HF cache, else None.""" + try: + from huggingface_hub import hf_hub_download + + from oellm.contrib.docvqa2026.datasets import HF_REPO, SPLIT + + return hf_hub_download( + HF_REPO, f"{SPLIT}.parquet", repo_type="dataset", local_files_only=True + ) + except Exception: + return None + + +@pytest.mark.skipif(_cached_val_parquet() is None, reason="val.parquet not cached") +class TestOracleOnRealData: + """Feeding the real answers back as predictions must score perfectly.""" + + @staticmethod + def _real_answers(): + import pyarrow.parquet as pq + + table = pq.ParquetFile(_cached_val_parquet()).read( + columns=["doc_category", "questions", "answers"] + ) + for row in table.to_pylist(): + answers = dict( + zip( + row["answers"]["question_id"], + row["answers"]["answer"], + strict=True, + ) + ) + for qid in row["questions"]["question_id"]: + yield row["doc_category"], answers[qid] + + def test_ground_truth_scores_perfectly(self): + import ast + + from oellm.contrib.docvqa2026.metrics import aggregate, score_prediction + + records = [] + for category, gt in self._real_answers(): + try: + parsed = ast.literal_eval(gt) + spoken = str(parsed[0]) if isinstance(parsed, list) else gt + except (ValueError, SyntaxError): + spoken = gt + correct, _, has_marker = score_prediction(f"FINAL ANSWER: {spoken}", gt) + records.append( + {"correct": correct, "doc_category": category, "has_marker": has_marker} + ) + + metrics = aggregate(records) + assert metrics["n_questions"] == 80 + assert metrics["n_categories"] == 8 + assert metrics["accuracy"] == 1.0 + assert metrics["macro_accuracy"] == 1.0 + + def test_dropping_the_marker_zeroes_the_same_answers(self): + """The 0.0 a non-compliant model earns is the scorer's first rule.""" + from oellm.contrib.docvqa2026.metrics import aggregate, score_prediction + + records = [ + { + "correct": score_prediction(gt, gt)[0], + "doc_category": category, + "has_marker": False, + } + for category, gt in self._real_answers() + ] + metrics = aggregate(records) + assert metrics["accuracy"] == 0.0 + assert metrics["format_compliance"] == 0.0 + + +class TestDocumentedQuirksHold: + """Behaviour the competition scorer depends on, stated explicitly.""" + + def test_missing_marker_is_always_wrong(self, scorer): + correct, extracted = scorer.evaluate_docvqa_prediction("4", "4") + assert correct is False + assert extracted == "4" + + def test_numeric_ground_truth_skips_the_anls_fallback(self, scorer): + assert scorer.evaluate_docvqa_prediction("FINAL ANSWER: four", "4")[0] is False + + def test_value_equal_unit_different_is_wrong(self, scorer): + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: 50 g", "50 kg")[0] is False + ) + + def test_unit_and_value_equal_is_right(self, scorer): + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: 50 kg", "50 kg")[0] is True + ) + + def test_textual_date_matches_iso_ground_truth(self, scorer): + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: Jan 1st 24", "2024-01-01")[0] + is True + ) + + def test_version_strings_compare_exactly(self, scorer): + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: 12.0.0", "12.0.0")[0] is True + ) + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: 12.0.1", "12.0.0")[0] + is False + ) + + def test_list_ground_truth_accepts_any_candidate(self, scorer): + gt = "['olive green', 'green', 'dark green']" + assert scorer.evaluate_docvqa_prediction("FINAL ANSWER: green", gt)[0] is True + assert scorer.evaluate_docvqa_prediction("FINAL ANSWER: blue", gt)[0] is False + + def test_articles_and_punctuation_are_normalised(self, scorer): + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: a wrench!", "wrench")[0] + is True + ) + + def test_anls_threshold_sits_at_0_90(self, scorer): + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: adjustablee", "adjustable")[ + 0 + ] + is True + ) + assert ( + scorer.evaluate_docvqa_prediction("FINAL ANSWER: wrenchh", "wrench")[0] + is False + ) + + def test_last_marker_wins(self, scorer): + correct, extracted = scorer.evaluate_docvqa_prediction( + "FINAL ANSWER: wrong\nFINAL ANSWER: 4", "4" + ) + assert extracted == "4" + assert correct is True + + def test_prompt_demands_the_marker(self, scorer): + assert "FINAL ANSWER:" in scorer.get_evaluation_prompt() From a0c4473cbd748285f8fc09f2e06f56fd45a881b5 Mon Sep 17 00:00:00 2001 From: islobozhan Date: Wed, 23 Sep 2026 13:38:05 +0200 Subject: [PATCH 38/44] [Base] Review fixes: correct results, scheduling and datasets; AudioBench checkpoints --- .github/sky/slurm-integration.yaml | 2 +- README.md | 4 +- containers/jupiter.def | 2 + containers/jureca.def | 2 + containers/leonardo.def | 2 + containers/slurm-ci.def | 2 + containers/snellius.def | 2 + docs/VENV.md | 4 +- oellm/config.py | 7 +- oellm/contrib/audiobench/README.md | 43 ++-- oellm/contrib/audiobench/adapter.py | 83 ++++++- oellm/contrib/audiobench/launch.py | 57 +++++ oellm/contrib/audiobench/suite.py | 127 ++++++----- oellm/main.py | 3 +- oellm/resources/task-groups.yaml | 9 +- oellm/resources/template.sbatch | 5 +- oellm/results.py | 280 ++++++++++++++++++------ oellm/scheduler.py | 90 ++++++-- oellm/task_groups.py | 14 +- tests/test_audiobench.py | 182 ++++++++++----- tests/test_collect_results.py | 156 ++++++++++++- tests/test_collection_and_scheduling.py | 13 +- tests/test_image_task_groups.py | 4 +- tests/test_reporter.py | 8 +- tests/test_schedule_evals.py | 5 +- tests/test_schedule_inputs.py | 112 ++++++++++ tests/test_task_groups.py | 22 ++ 27 files changed, 1000 insertions(+), 240 deletions(-) create mode 100644 oellm/contrib/audiobench/launch.py create mode 100644 tests/test_schedule_inputs.py diff --git a/.github/sky/slurm-integration.yaml b/.github/sky/slurm-integration.yaml index d5baa4a9..73ea3f8d 100644 --- a/.github/sky/slurm-integration.yaml +++ b/.github/sky/slurm-integration.yaml @@ -127,7 +127,7 @@ run: | lm-eval torch transformers accelerate "datasets<4.0.0" UV_TOOL_DIR="$EVAL_BASE_DIR/.uv-tools" UV_TOOL_BIN_DIR="$EVAL_BASE_DIR/.venv/bin" \ uv tool install --python 3.12 \ - --with "langcodes[data]" --with "pillow" \ + --with "langcodes[data]" --with "pillow" --with "xxhash<4" \ "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git" fi diff --git a/README.md b/README.md index 23054fab..b5cc7940 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,7 @@ If you use custom tasks via `--tasks` that are not in the task groups registry, ## Collecting Results -After evaluations complete, collect results into a CSV. `collect` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs. Alongside the CSV/Markdown tables it writes `eval_results.json` — a versioned envelope that embeds each run's provenance (engine versions, model revisions, quantization, submitter), which is also the ingestion format for the ELLIOT evaluation dashboard: +After evaluations complete, collect results into a CSV. `collect` **recursively** searches the given directory for every `jobs.csv` file and every `.json` result file, so you can point it at a top-level output folder that contains many sub-runs. Alongside the CSV/Markdown tables it writes `eval_results.json` — a versioned envelope that embeds the provenance (engine versions, model revisions, quantization, submitter) of the runs its rows came from, and gives each row its run, `--limit`, quantization and evaluation time; it is also the ingestion format for the ELLIOT evaluation dashboard. When the same result exists from a full evaluation and from a `--limit` test run, the full one is kept; limited rows are marked: ``` output/ @@ -382,7 +382,7 @@ oellm-eval collect /path/to/eval-output-dir --check --output-csv results.csv Three output files are written next to your `--output-csv` path: the CSV (raw metric per row), a versioned JSON envelope, and a Markdown table with metrics normalized to a 0–100 scale. -All `jobs.csv` files found under `results_dir` are merged into one; if the same `(model_path, task_path, n_shot)` row appears in multiple files the later-sorted entry wins (override duplicates). The merged jobs list is then compared against all `.json` result files found recursively. +All `jobs.csv` files found under `results_dir` are merged into one; if the same `(model_path, task_path, n_shot)` row appears in multiple files, one scheduled without `--limit` wins (it only counts as done once a full result exists), otherwise the later-sorted entry. The merged jobs list is then compared against all `.json` result files found recursively. The `--check` flag outputs a `results_missing.csv` that can be used to re-schedule failed jobs: diff --git a/containers/jupiter.def b/containers/jupiter.def index 349f2a32..a4233713 100644 --- a/containers/jupiter.def +++ b/containers/jupiter.def @@ -21,9 +21,11 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 export UV_TOOL_DIR=/opt/uv-tools # Install lighteval without [multilingual] extras first, then add multilingual deps # sudachipy (spacy[ja] dep) has no ARM64 wheel, so skip Japanese support + # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def) uv tool install --python 3.12 \ --with "pillow" \ --with "torch<2.9" \ + --with "xxhash<4" \ "lighteval @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" uv pip install --system --break-system-packages "spacy>=3.0.0,<4.0.0" jieba "underthesea>=6.0.0" uv pip install --system --break-system-packages nltk diff --git a/containers/jureca.def b/containers/jureca.def index d59a99f8..fa2cc22d 100644 --- a/containers/jureca.def +++ b/containers/jureca.def @@ -24,9 +24,11 @@ From: nvcr.io/nvidia/pytorch:25.06-py3 nltk # lighteval as isolated tool (avoids dependency conflicts) + # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def) uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ + --with "xxhash<4" \ "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time diff --git a/containers/leonardo.def b/containers/leonardo.def index 4b69f86b..8455346d 100644 --- a/containers/leonardo.def +++ b/containers/leonardo.def @@ -24,9 +24,11 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 nltk # lighteval as isolated tool (avoids dependency conflicts) + # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def) uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ + --with "xxhash<4" \ "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time diff --git a/containers/slurm-ci.def b/containers/slurm-ci.def index ad1e3145..c193c85a 100644 --- a/containers/slurm-ci.def +++ b/containers/slurm-ci.def @@ -24,9 +24,11 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 nltk # lighteval as isolated tool (avoids dependency conflicts) + # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def) uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ + --with "xxhash<4" \ "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time diff --git a/containers/snellius.def b/containers/snellius.def index 7b572fe2..68e7b695 100644 --- a/containers/snellius.def +++ b/containers/snellius.def @@ -24,9 +24,11 @@ From: nvcr.io/nvidia/pytorch:25.10-py3 nltk # lighteval as isolated tool (avoids dependency conflicts) + # xxhash<4: 4.0 rejects str input, which lighteval hashes (see lumi.def) uv tool install --python 3.12 \ --with "langcodes[data]" \ --with "pillow" \ + --with "xxhash<4" \ "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" # Pre-load lighteval registry to trigger tinyBenchmarks data download at build time diff --git a/docs/VENV.md b/docs/VENV.md index f5324094..f108fd4b 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -48,10 +48,10 @@ uv pip install --python /path/to/.venv/bin/python -e /path/to/lmms-eval # 3. Install oellm-cli with engine extras uv pip install --python /path/to/.venv/bin/python -e '.[text,image,audio]' -# 4. Install lighteval as an isolated uv tool (datasets version conflict) +# 4. Install lighteval as an isolated uv tool (datasets version conflict; xxhash 4 breaks it) UV_TOOL_DIR=/path/to/.uv-tools UV_TOOL_BIN_DIR=/path/to/.venv/bin \ uv tool install --python 3.12 \ - --with "langcodes[data]" --with "pillow" \ + --with "langcodes[data]" --with "pillow" --with "xxhash<4" \ "lighteval[multilingual] @ git+https://github.com/huggingface/lighteval.git@64f4f5ae173626509fad6e477ca4ee56ebb26129" ``` diff --git a/oellm/config.py b/oellm/config.py index fdd68500..8eace906 100644 --- a/oellm/config.py +++ b/oellm/config.py @@ -116,7 +116,6 @@ def from_yaml(cls, path: str | Path) -> EvalConfig: - "Qwen/Qwen2-VL-7B" task_groups: - "image-vqa" - n_shot: 0 trust_remote_code: true venv_path: "~/elliot-venv" slurm: @@ -402,6 +401,12 @@ def validate(self) -> None: if self.tasks and not self.n_shot: raise ValueError("n_shot is required when specifying individual tasks.") + if self.n_shot and self.task_groups and not self.tasks: + raise ValueError( + "n_shot applies to tasks only; task groups set their own shots " + "(see oellm-eval list-tasks)." + ) + if self.n_shot: for s in self.n_shot: if not isinstance(s, int) or s < 0: diff --git a/oellm/contrib/audiobench/README.md b/oellm/contrib/audiobench/README.md index 6276acdd..5e35a8d9 100644 --- a/oellm/contrib/audiobench/README.md +++ b/oellm/contrib/audiobench/README.md @@ -154,24 +154,35 @@ vs `lmms_eval`) — no silent averaging. ## Supported model adapters AudioBench dispatches on a fixed list of literal `model_name` strings -(see `$AUDIOBENCH_DIR/src/model.py`); each loader under `model_src/` -fetches its own HF repo. Arbitrary HF checkpoints are not supported — -only the variants below: - -| Model path substring (lowered) | AudioBench `model_name` (literal) | -|------------------------------------------------|-------------------------------------------| -| `qwen2-audio-7b-instruct` / `qwen2_audio_7b_instruct` | `Qwen2-Audio-7B-Instruct` | -| `qwen-audio-chat` / `qwen_audio_chat` | `Qwen-Audio-Chat` | -| `salmonn` | `SALMONN_7B` | -| `meralion-audiollm` / `meralion_audiollm` | `MERaLiON-AudioLLM-Whisper-SEA-LION` | -| `whisper-large-v3` / `whisper_large_v3` | `whisper_large_v3` | -| `whisper-large-v2` / `whisper_large_v2` | `whisper_large_v2` | -| `phi-4-multimodal` / `phi_4_multimodal` | `phi_4_multimodal_instruct` | -| `seallms-audio-7b` / `seallms_audio_7b` | `seallms_audio_7b` | -| `wavllm` | `WavLLM_fairseq` | -| (anything else) | error — no generic loader upstream | +(see `$AUDIOBENCH_DIR/src/model.py`), and each loader under `model_src/` +reads its weights location from a variable (`model_path = "Qwen/Qwen2-Audio-7B-Instruct"`). +The plugin's `launch.py` points that variable at your model, so checkpoints of +these families are evaluated with the family's AudioBench prompts and loader: + +| Model path substring (lowered) | AudioBench `model_name` (literal) | Your checkpoints | +|------------------------------------------------|-------------------------------------------|------------------| +| `qwen2-audio-7b-instruct` / `qwen2_audio_7b_instruct` | `Qwen2-Audio-7B-Instruct` | yes | +| `qwen-audio-chat` / `qwen_audio_chat` | `Qwen-Audio-Chat` | yes | +| `meralion-audiollm` / `meralion_audiollm` | `MERaLiON-AudioLLM-Whisper-SEA-LION` | yes | +| `whisper-large-v3` / `whisper_large_v3` | `whisper_large_v3` | yes | +| `whisper-large-v2` / `whisper_large_v2` | `whisper_large_v2` | yes | +| `phi-4-multimodal` / `phi_4_multimodal` | `phi_4_multimodal_instruct` | yes | +| `salmonn` | `SALMONN_7B` | stock only | +| `seallms-audio-7b` / `seallms_audio_7b` | `seallms_audio_7b` | stock only | +| `wavllm` | `WavLLM_fairseq` | stock only | +| (anything else) | error — no generic loader upstream | | + +A local checkpoint folder whose path doesn't name its family is recognised +from `config.json`: `Qwen2AudioForConditionalGeneration`, +`MERaLiONForConditionalGeneration`, `WhisperForConditionalGeneration` (any +size, run with the `whisper_large_v3` loader) and `Phi4MMForCausalLM`. +SeaLLMs-Audio loads a hard-coded repo id, SALMONN a multi-file layout inside +the clone and WavLLM a fairseq script, so checkpoints of those are refused. To override detection, pass the literal AudioBench key as a suffix: `audiobench:Qwen2-Audio-7B-Instruct`. Case is preserved end-to-end (AudioBench's match is case-sensitive). +Each run writes AudioBench's predictions and score file to its own temporary +folder, not the clone's `log_for_all_models/`, so parallel runs of one family +cannot overwrite or reuse each other's scores. diff --git a/oellm/contrib/audiobench/adapter.py b/oellm/contrib/audiobench/adapter.py index 66a9655a..1c2cfc04 100644 --- a/oellm/contrib/audiobench/adapter.py +++ b/oellm/contrib/audiobench/adapter.py @@ -1,14 +1,7 @@ -"""AudioBench model adapter. +"""AudioBench model adapter: maps a model path to AudioBench's ``--model_name``. -Maps a HuggingFace model path to AudioBench's literal ``--model_name`` value. - -AudioBench's ``Model`` class (in ``$AUDIOBENCH_DIR/src/model.py``) dispatches -on **exact-string** match against a fixed list — there is no family-level -indirection and no fallback. Each supported model has a hardcoded loader -under ``model_src/`` that loads its own HF repo internally; AudioBench -**cannot evaluate arbitrary HF checkpoints**, only the variants it knows -about. If we can't map the user's ``model_path`` to one of those literals, -we return ``None`` and ``suite.run`` raises a clear error. +For families in :data:`CHECKPOINT_VARIABLE`, ``launch.py`` loads the given +checkpoint instead of the stock weights; the others run their stock model only. """ from __future__ import annotations @@ -61,9 +54,77 @@ def to_contrib_flags(self) -> str | None: for key, needles in _PATTERNS: if any(n in lowered for n in needles): return key - return None + return _family_from_config(self._path) def detect_audiobench_model_type(model_path: str) -> str | None: """Convenience wrapper around :meth:`AudioBenchModelAdapter.to_contrib_flags`.""" return AudioBenchModelAdapter(model_path).to_contrib_flags() + + +# Family -> (model_src module, variable holding its weights location). +# SeaLLMs-Audio, SALMONN and WavLLM hard-code theirs: stock models only. +CHECKPOINT_VARIABLE: dict[str, tuple[str, str]] = { + "Qwen2-Audio-7B-Instruct": ("qwen2_audio_7b_instruct", "model_path"), + "Qwen-Audio-Chat": ("qwen_audio_chat", "model_path"), + "MERaLiON-AudioLLM-Whisper-SEA-LION": ( + "meralion_audiollm_whisper_sea_lion", + "repo_id", + ), + "whisper_large_v3": ("whisper_large_v3", "whisper_model_path"), + "whisper_large_v2": ("whisper_large_v2", "whisper_model_path"), + "phi_4_multimodal_instruct": ("phi_4_multimodal_instruct", "model_path"), +} + +# config.json architecture -> family, for checkpoints whose path doesn't name it. +_ARCHITECTURES: dict[str, str] = { + "Qwen2AudioForConditionalGeneration": "Qwen2-Audio-7B-Instruct", + "MERaLiONForConditionalGeneration": "MERaLiON-AudioLLM-Whisper-SEA-LION", + "WhisperForConditionalGeneration": "whisper_large_v3", # same loader as v2 + "Phi4MMForCausalLM": "phi_4_multimodal_instruct", +} + + +def _family_from_config(model_path: str) -> str | None: + import json + from pathlib import Path + + config = Path(model_path).expanduser() / "config.json" + try: + architectures = json.loads(config.read_text()).get("architectures") or [] + except (OSError, ValueError, AttributeError): + return None + return next((_ARCHITECTURES[a] for a in architectures if a in _ARCHITECTURES), None) + + +# Name (last path part, "-" = "_") of each family's stock model. +_STOCK_NAMES: dict[str, tuple[str, ...]] = { + "Qwen2-Audio-7B-Instruct": ("qwen2-audio-7b-instruct",), + "Qwen-Audio-Chat": ("qwen-audio-chat",), + "SALMONN_7B": ("salmonn", "salmonn-7b"), + "MERaLiON-AudioLLM-Whisper-SEA-LION": ("meralion-audiollm-whisper-sea-lion",), + "whisper_large_v3": ("whisper-large-v3",), + "whisper_large_v2": ("whisper-large-v2",), + "phi_4_multimodal_instruct": ("phi-4-multimodal-instruct",), + "seallms_audio_7b": ("seallms-audio-7b",), + "WavLLM_fairseq": ("wavllm", "wavllm-fairseq"), +} + + +def is_stock_model(model_path: str, key: str) -> bool: + """*model_path* names the stock model, not a local checkpoint or a fine-tune.""" + from pathlib import Path + + if model_path.startswith(("/", "~", ".")) or Path(model_path).expanduser().exists(): + return False + name = model_path.rstrip("/").rsplit("/", 1)[-1].lower().replace("_", "-") + stock = {key.lower().replace("_", "-"), *_STOCK_NAMES.get(key, ())} + return name in stock + + +def stock_only_message(model_path: str, key: str) -> str: + return ( + f"AudioBench cannot load {model_path!r} with its {key} loader: that " + f"loader only runs the stock model (its name is {_STOCK_NAMES[key][0]}). " + f"Checkpoints work for: {', '.join(CHECKPOINT_VARIABLE)}." + ) diff --git a/oellm/contrib/audiobench/launch.py b/oellm/contrib/audiobench/launch.py new file mode 100644 index 00000000..edda3ce4 --- /dev/null +++ b/oellm/contrib/audiobench/launch.py @@ -0,0 +1,57 @@ +"""Run one AudioBench evaluation, optionally on your own checkpoint. + +Points the loader's weights variable at ``--checkpoint`` and writes AudioBench's +files to ``--log-dir``. Run from the AudioBench clone: some loaders use relative paths. +""" + +from __future__ import annotations + +import argparse +import importlib +import os +import sys +from pathlib import Path + + +def main(argv: list[str] | None = None) -> None: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--audiobench-dir", required=True) + p.add_argument("--log-dir", required=True) + p.add_argument("--dataset-name", required=True) + p.add_argument("--model-name", required=True, help="AudioBench dispatch key") + p.add_argument("--metrics", required=True) + p.add_argument("--number-of-samples", type=int, default=-1) + p.add_argument("--module", help="model_src module whose weights to replace") + p.add_argument("--variable", help="module variable holding the weights location") + p.add_argument("--checkpoint", help="checkpoint to load instead") + args = p.parse_args(argv) + + src = Path(args.audiobench_dir).resolve() / "src" + # AudioBench imports its modules by bare name; keep this folder off the path. + here = str(Path(__file__).resolve().parent) + sys.path[:] = [str(src)] + [entry for entry in sys.path if entry != here] + os.chdir(args.audiobench_dir) + + import main_evaluate + + main_evaluate.file_save_folder = str(Path(args.log_dir).resolve()) + + if args.checkpoint: + module = importlib.import_module(f"model_src.{args.module}") + stock = getattr(module, args.variable) + setattr(module, args.variable, args.checkpoint) + print( + f"AudioBench {args.model_name}: loading {args.checkpoint} instead of {stock}" + ) + + main_evaluate.main( + dataset_name=args.dataset_name, + model_name=args.model_name, + metrics=args.metrics, + overwrite=True, + number_of_samples=args.number_of_samples, + ) + + +if __name__ == "__main__": + main() diff --git a/oellm/contrib/audiobench/suite.py b/oellm/contrib/audiobench/suite.py index 7ec28e46..c7734ed2 100644 --- a/oellm/contrib/audiobench/suite.py +++ b/oellm/contrib/audiobench/suite.py @@ -1,11 +1,10 @@ """AudioBench contrib suite — plugin protocol implementation. AudioBench is not pip-installable (upstream has no build backend and uses -bare imports like ``from dataset import ...``), so :func:`run` invokes its -``src/main_evaluate.py`` entry point as a subprocess with ``cwd`` set to -``$AUDIOBENCH_DIR``. :func:`run` then re-shapes AudioBench's result JSON -into a lmms-eval-compatible payload that :func:`oellm.main.collect_results` -can parse unchanged. +bare imports like ``from dataset import ...``), so :func:`run` runs it through +``launch.py`` in a subprocess with ``cwd`` set to ``$AUDIOBENCH_DIR``, then +re-shapes AudioBench's result JSON into a lmms-eval-compatible payload that +:func:`oellm.main.collect_results` can parse unchanged. """ from __future__ import annotations @@ -14,6 +13,8 @@ import logging import os import subprocess +import sys +import tempfile from pathlib import Path from oellm.contrib.audiobench.task import ( @@ -104,10 +105,21 @@ def detect_model_flags(model_path: str) -> str | None: model family — :func:`run` then raises a clear error. AudioBench has no generic loader, so silently falling back to a fictitious key would just move the error deeper inside the subprocess. + + Raises ``ValueError`` for a checkpoint of a stock-only family. """ - from oellm.contrib.audiobench.adapter import AudioBenchModelAdapter + from oellm.contrib.audiobench.adapter import ( + CHECKPOINT_VARIABLE, + AudioBenchModelAdapter, + is_stock_model, + stock_only_message, + ) - return AudioBenchModelAdapter(model_path).to_contrib_flags() + key = AudioBenchModelAdapter(model_path).to_contrib_flags() + if key is not None and key not in CHECKPOINT_VARIABLE: + if not is_stock_model(model_path, key): + raise ValueError(stock_only_message(model_path, key)) + return key def run( @@ -147,46 +159,66 @@ def run( f"model. AudioBench dispatches on a fixed list of literal " f"model_name strings (Qwen2-Audio-7B-Instruct, SALMONN_7B, " f"whisper_large_v3, …) — see oellm/contrib/audiobench/adapter.py. " - f"AudioBench cannot evaluate arbitrary HF checkpoints; it loads " - f"its own hardcoded HF repos per model family." + f"Pass a path that names the family, or a local checkpoint folder " + f"with its config.json." ) model_key = model_flags # AudioBench's dispatch key, e.g. "Qwen2-Audio-7B-Instruct" - cmd = [ - "python", - "src/main_evaluate.py", - "--dataset_name", - spec.upstream_name, - "--model_name", - model_key, - "--metrics", - spec.upstream_metric, - # Force re-eval — AudioBench skips by default if a stale score file - # already exists under log_for_all_models/. - "--overwrite", - "True", - ] - - limit = env.get("LIMIT", "").strip() - if limit: - cmd.extend(["--number_of_samples", str(limit)]) - - logger.info("AudioBench cmd: %s (cwd=%s)", " ".join(cmd), ab_dir) - completed = subprocess.run( - cmd, - cwd=ab_dir, - env=env, - check=False, + from oellm.contrib.audiobench.adapter import ( + CHECKPOINT_VARIABLE, + is_stock_model, + stock_only_message, ) - if completed.returncode != 0: - raise RuntimeError( - f"AudioBench exited with code {completed.returncode} for " - f"task={task!r} model={model_path!r} (dispatch key={model_key!r})" + + load_checkpoint = not is_stock_model(model_path, model_key) + if load_checkpoint and model_key not in CHECKPOINT_VARIABLE: + raise RuntimeError(stock_only_message(model_path, model_key)) + + # Own folder per run: AudioBench names its files by family, not by model. + with tempfile.TemporaryDirectory(prefix="audiobench_") as log_dir: + cmd = [ + sys.executable, + str(Path(__file__).with_name("launch.py")), + "--audiobench-dir", + ab_dir, + "--log-dir", + log_dir, + "--dataset-name", + spec.upstream_name, + "--model-name", + model_key, + "--metrics", + spec.upstream_metric, + ] + if load_checkpoint: + module, variable = CHECKPOINT_VARIABLE[model_key] + cmd += [ + "--module", + module, + "--variable", + variable, + "--checkpoint", + model_path, + ] + + limit = env.get("LIMIT", "").strip() + if limit: + cmd += ["--number-of-samples", str(limit)] + + logger.info("AudioBench cmd: %s (cwd=%s)", " ".join(cmd), ab_dir) + completed = subprocess.run( + cmd, + cwd=ab_dir, + env=env, + check=False, ) + if completed.returncode != 0: + raise RuntimeError( + f"AudioBench exited with code {completed.returncode} for " + f"task={task!r} model={model_path!r} (dispatch key={model_key!r})" + ) - metrics = _extract_metrics( - audiobench_dir=Path(ab_dir), model_key=model_key, spec=spec - ) + metrics = _extract_metrics(log_dir=Path(log_dir), model_key=model_key, spec=spec) _write_lmms_shaped_json( output_path=output_path, model_path=model_path, @@ -199,20 +231,13 @@ def run( def _extract_metrics( *, - audiobench_dir: Path, + log_dir: Path, model_key: str, spec: AudioBenchTaskSpec, ) -> dict[str, float]: - """Read AudioBench's score file from its hardcoded output path. - - AudioBench writes to ``$cwd/log_for_all_models//__score.json`` - (see ``main_evaluate.py:118``). Path is fixed — there is no ``--log_dir``. - """ + """Read AudioBench's score file for this run from *log_dir*.""" score_file = ( - audiobench_dir - / "log_for_all_models" - / model_key - / f"{spec.upstream_name}_{spec.upstream_metric}_score.json" + log_dir / model_key / f"{spec.upstream_name}_{spec.upstream_metric}_score.json" ) if not score_file.exists(): raise RuntimeError( diff --git a/oellm/main.py b/oellm/main.py index 9e1f1c43..d8af6dac 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -68,7 +68,8 @@ def schedule_evals( tasks: A string of comma-separated task names (lm_eval) or paths. Requires `n_shot` to be provided. Tasks here are assumed to be lm_eval unless otherwise handled via CSV. task_groups: A string of comma-separated task group names defined in `task-groups.yaml`. - Each group expands into concrete (task, n_shots, suite) entries; `n_shot` is ignored for groups. + Each group expands into concrete (task, n_shots, suite) entries and sets its own shots; + `n_shot` applies to `tasks` only. Given together, `tasks` and `task_groups` are both scheduled. A group (or super_group) may be scoped to one or more languages with a bracket, e.g. `--task-groups "oellm-multilingual[deu_Latn]"` or `--task-groups "sib200-eu[fra_Latn|deu_Latn],flores-200-eu-to-eng[deu_Latn]"`. Bracketed diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 0cee8873..d4cac24a 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -252,7 +252,7 @@ task_groups: - task: xwinograd dataset: Muennighoff/xwinograd - task: xcopa - dataset: cambridgeltl/xcopa + dataset: xcopa # the id lm-eval loads; offline caches are keyed by it - task: xstorycloze dataset: juletxara/xstory_cloze @@ -320,7 +320,7 @@ task_groups: dataset: lmms-lab/MMBench - task: mmmu_val metric: mmmu_acc - dataset: MMMU/MMMU + dataset: lmms-lab/MMMU - task: chartqa metric: relaxed_overall dataset: lmms-lab/ChartQA @@ -370,7 +370,7 @@ task_groups: n_shots: [0] tasks: - task: mmmu_val - dataset: MMMU/MMMU + dataset: lmms-lab/MMMU image-chartqa: description: "ChartQA chart question answering via lmms-eval" @@ -1002,7 +1002,8 @@ task_groups: tasks: - task: "arc_challenge_mt_{lang}" subset: "{lang}" - - task: arc_challenge_mt_is # no subset in the source dataset + - task: arc_challenge_mt_is + dataset: mideind/icelandic-arc-challenge # not in LumiOpen/arc_challenge_mt hellaswag-eu: description: "Okapi HellaSwag multilingual (16 EU languages, 0-shot)." diff --git a/oellm/resources/template.sbatch b/oellm/resources/template.sbatch index 207b0fc5..cf100b3a 100644 --- a/oellm/resources/template.sbatch +++ b/oellm/resources/template.sbatch @@ -30,7 +30,8 @@ LM_EVAL_BATCH_SIZE="{lm_eval_batch_size}" # Compute nodes are air-gapped — every dataset must be cache-resolved. export HF_HOME=$HF_HOME -export HF_DATASETS_CACHE="$HF_HOME/datasets" +# The cache the pre-download filled: an exported HF_DATASETS_CACHE, else the default. +export HF_DATASETS_CACHE="${{HF_DATASETS_CACHE:-$HF_HOME/datasets}}" export HF_HUB_OFFLINE={hf_hub_offline} # HF_DATASETS_OFFLINE covers `dl_manager.download(arbitrary_url)` (external # image/video hosts) which HF_HUB_OFFLINE does not. @@ -38,7 +39,7 @@ export HF_DATASETS_OFFLINE={hf_hub_offline} # lmms-eval auto-redirects "remote" (GPFS/Lustre) caches to a node-local # /tmp by default; set this to force it to use the shared cache instead. # See lmms_eval/api/task.py::_resolve_hf_datasets_cache_dir. -export LMMS_EVAL_DATASETS_CACHE="$HF_HOME/datasets" +export LMMS_EVAL_DATASETS_CACHE="$HF_DATASETS_CACHE" # Path to the shared Singularity image that contains all runtime deps (container mode) export EVAL_SIF_PATH="$EVAL_BASE_DIR/$EVAL_CONTAINER_IMAGE" diff --git a/oellm/results.py b/oellm/results.py index deab50f6..715d2e78 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -5,7 +5,7 @@ import json import logging import subprocess -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta, timezone from pathlib import Path import pandas as pd @@ -82,8 +82,18 @@ TASK_METRIC_SCALE_OVERRIDES: dict[tuple[str, str], float] = { ("squadv2", "f1"): 100.0, ("voicebench_commoneval", "llm_as_judge_eval"): 5.0, + # lmms-eval 45c766f: alpaca_audio and openhermes report 0–100, air_bench_chat 1–10. + ("alpaca_audio", "gpt_eval"): 100.0, + ("openhermes", "gpt_eval"): 100.0, + ("air_bench_chat_sound", "gpt_eval"): 10.0, + ("air_bench_chat_music", "gpt_eval"): 10.0, + ("air_bench_chat_speech", "gpt_eval"): 10.0, + ("air_bench_chat_mixed", "gpt_eval"): 10.0, } +# Counters in engine output, never scores. +_NON_METRIC_KEYS = {"alias", "name", " ", "", "sample_len", "sample_count", "samples"} + def _normalize_to_100( value: float | None, metric_name: str | None, task_name: str | None = None @@ -153,14 +163,21 @@ def _first_matching_prefix(d: dict, prefix: str) -> tuple[float | None, str | No return val, key # Last resort: pick the first numeric non-stderr value (catches lmms-eval - # benchmarks with non-standard metric names like mme_cognition_score) - for k, v in result_dict.items(): - if ( - isinstance(v, (int, float)) - and "stderr" not in k - and k not in ("alias", " ", "") - ): + # benchmarks with non-standard metric names like mme_cognition_score). + # Prefer engine metric keys ("name,filter") over bare keys. + candidates = [ + (k, v) + for k, v in result_dict.items() + if isinstance(v, (int, float)) + and not isinstance(v, bool) + and "stderr" not in k + and k not in _NON_METRIC_KEYS + ] + for k, v in candidates: + if "," in k: return float(v), k + for k, v in candidates: + return float(v), k return None, None @@ -172,7 +189,7 @@ def _extract_all_metrics(result_dict: dict) -> list[tuple[str, float]]: key = raw_key.split("/", 1)[1] if "/" in raw_key else raw_key if isinstance(value, bool) or not isinstance(value, (int, float)): continue - if key in seen or key in ("alias", " ", ""): + if key in seen or key in _NON_METRIC_KEYS: continue seen.add(key) pairs.append((key, float(value))) @@ -304,6 +321,80 @@ def _try_contrib_parse(data: dict) -> tuple[str, str, int, dict] | None: return None +def _run_of( + path: Path, cache: dict[Path, dict | None], stop: Path +) -> tuple[Path | None, dict | None]: + """Nearest folder above *path*, up to *stop*, with a provenance.json.""" + stop = stop.resolve() + for folder in path.resolve().parents: + if folder not in cache: + sidecar = folder / "provenance.json" + loaded = None + if sidecar.is_file(): + try: + loaded = json.loads(sidecar.read_text()) + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: + logging.warning(f"Unreadable provenance sidecar {sidecar}: {e}") + cache[folder] = loaded if isinstance(loaded, dict) else None + if cache[folder] is not None: + return folder, cache[folder] + if folder == stop: + break + return None, None + + +def _sample_limit(data: dict, provenance: dict | None) -> int | float | None: + """The --limit behind a result, None for a full evaluation.""" + config = data.get("config") + config_general = data.get("config_general") + for value in ( + (provenance or {}).get("limit"), + config.get("limit") if isinstance(config, dict) else None, + config_general.get("max_samples") if isinstance(config_general, dict) else None, + ): + if isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0: + return int(value) if float(value).is_integer() else value + return None + + +def _quantization(data: dict) -> str | None: + """Quantization the engine used; lighteval and contrib suites ignore the flag.""" + config = data.get("config") + args = config.get("model_args") if isinstance(config, dict) else None + text = json.dumps(args) if isinstance(args, dict) else str(args or "") + for bits in ("4bit", "8bit"): + if f"load_in_{bits}=True" in text or f'"load_in_{bits}": true' in text: + return bits + return None + + +# lmms-eval's default timezone for "date"; the job script doesn't pass --timezone. +_LMMS_EVAL_TZ = timezone(timedelta(hours=8)) + + +def _evaluated_at(data: dict, path: Path) -> str: + """Evaluation time in UTC: the engine's date, else the file's modification time.""" + date = data.get("date") + moment = None + if isinstance(date, (int, float)) and not isinstance(date, bool): + try: + moment = datetime.fromtimestamp(date, UTC) + except (OverflowError, OSError, ValueError): + moment = None + elif isinstance(date, str): + try: + moment = ( + datetime.strptime(date, "%Y%m%d_%H%M%S") + .replace(tzinfo=_LMMS_EVAL_TZ) + .astimezone(UTC) + ) + except ValueError: + moment = None + if moment is None: + moment = datetime.fromtimestamp(path.stat().st_mtime, UTC) + return moment.isoformat(timespec="seconds") + + def collect_results( results_dir: str, output_csv: str = "eval_results.csv", @@ -347,19 +438,14 @@ def collect_results( # on filesystem enumeration order. json_files.sort(key=lambda p: (p.stat().st_mtime, str(p))) - # Run-provenance sidecars written by the scheduler (schedule-time config, - # resolved model revisions, template knobs). Embedded verbatim in the - # results JSON envelope so a collected number can be traced to its run. - run_provenance: list[dict] = [] - for _prov in sorted(results_path.rglob("provenance.json")): - try: - _pdata = json.loads(_prov.read_text()) - except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: - logging.warning(f"Unreadable provenance sidecar {_prov}: {e}") - continue - if isinstance(_pdata, dict): - _pdata["_path"] = str(_prov) - run_provenance.append(_pdata) + # provenance.json of each run folder, keyed by folder. + provenance_cache: dict[Path, dict | None] = {} + run_root = results_path.resolve().parent + + def _needs_full(jobs_csv: Path) -> bool: + """Scheduled without --limit according to its provenance.json.""" + provenance = _run_of(jobs_csv, provenance_cache, run_root)[1] + return provenance is not None and _sample_limit({}, provenance) is None if not json_files: logging.warning(f"No JSON files found in {results_dir}") @@ -386,17 +472,27 @@ def collect_results( f"{[str(p) for p in jobs_csv_paths]}" ) jobs_df = pd.concat( - [pd.read_csv(p) for p in jobs_csv_paths], ignore_index=True + [ + pd.read_csv(p).assign(_needs_full=_needs_full(p)) + for p in jobs_csv_paths + ], + ignore_index=True, ) dup_cols = [ c for c in ("model_path", "task_path", "n_shot") if c in jobs_df.columns ] if dup_cols: - jobs_df = jobs_df.drop_duplicates(subset=dup_cols, keep="last") + # A job scheduled with and without --limit counts as full. + jobs_df = jobs_df.sort_values( + "_needs_full", kind="stable" + ).drop_duplicates(subset=dup_cols, keep="last") logging.info(f"Merged jobs.csv: {len(jobs_df)} unique scheduled jobs") rows = [] - completed_jobs = set() + # (model, task, n_shot) -> a full result exists + completed_jobs: dict[tuple, bool] = {} + file_ctx: dict = {} + row_run: dict[int, Path] = {} def _metric_pairs(task_name: str, result_dict: dict) -> list[tuple[str, float]]: if fetch_all_metrics: @@ -407,19 +503,29 @@ def _metric_pairs(task_name: str, result_dict: dict) -> list[tuple[str, float]]: return [(metric_name if metric_name is not None else "", performance)] def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None: - for metric_name, performance in pairs: - rows.append( - { - "model_name": model, - "task": task, - "n_shot": n_shot, - "performance": performance, - "performance_normalized": _normalize_to_100( - performance, metric_name, task - ), - "metric_name": metric_name, - } + if check and pairs: + key = (model, task, n_shot) + completed_jobs[key] = completed_jobs.get(key, False) or ( + file_ctx["limit"] is None ) + for metric_name, performance in pairs: + row = { + "model_name": model, + "task": task, + "n_shot": n_shot, + "performance": performance, + "performance_normalized": _normalize_to_100( + performance, metric_name, task + ), + "metric_name": metric_name, + "limit": file_ctx["limit"], + "quantization": file_ctx["quantization"], + "evaluated_at": file_ctx["evaluated_at"], + "run": file_ctx["run_dir"].name if file_ctx["run_dir"] else None, + } + rows.append(row) + if file_ctx["run_dir"] is not None: + row_run[id(row)] = file_ctx["run_dir"] for json_file in json_files: # Provenance sidecars are consumed separately above, not result files. @@ -446,6 +552,14 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None ) continue + run_dir, provenance = _run_of(json_file, provenance_cache, run_root) + file_ctx.update( + run_dir=run_dir, + limit=_sample_limit(data, provenance), + quantization=_quantization(data), + evaluated_at=_evaluated_at(data, json_file), + ) + # First-chance: a contrib suite may claim this file outright via its # parse_results() protocol member and own the format end-to-end. _contrib_parsed = _try_contrib_parse(data) @@ -453,8 +567,6 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None _c_model, _c_task, _c_n_shot, _c_metrics = _contrib_parsed _c_pairs = _metric_pairs(_c_task, _c_metrics) if _c_pairs: - if check: - completed_jobs.add((_c_model, _c_task, _c_n_shot)) _emit(_c_model, _c_task, _c_n_shot, _c_pairs) else: logging.warning( @@ -546,8 +658,6 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None n_shot = parsed_n _g_pairs = _metric_pairs(group_name, group_results) if _g_pairs: - if check: - completed_jobs.add((model_name, group_name, n_shot)) _emit(model_name, group_name, n_shot, _g_pairs) else: # Metric-less aggregate: descend into child groups so @@ -647,8 +757,6 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None pairs = _metric_pairs(task_name_clean, task_results) if pairs: - if check: - completed_jobs.add((model_name, task_name_clean, n_shot)) _emit(model_name, task_name_clean, n_shot, pairs) else: # Log missing metrics — for lmms-eval tasks this often means @@ -673,30 +781,56 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None return if rows: - # Drop duplicate (model, task, n_shot, metric) rows, keeping the last - # occurrence. Dedup the row list directly (not just the DataFrame) so - # the CSV, JSON, and Markdown outputs stay consistent and None values - # in performance_normalized survive (a pandas round-trip would coerce - # them to NaN and break the JSON envelope). - _deduped: dict[tuple, dict] = {} - for _row in rows: + # One row per (model, task, n_shot, metric): a full evaluation beats a + # --limit run, then the newest wins. Dedup the list, not a DataFrame, + # so None in performance_normalized doesn't become NaN. + _deduped: dict[tuple, tuple[tuple, dict]] = {} + for _i, _row in enumerate(rows): _key = ( _row.get("model_name"), _row.get("task"), _row.get("n_shot"), _row.get("metric_name"), ) - if _key in _deduped and _deduped[_key].get("performance") != _row.get( - "performance" - ): - logging.warning( - f"Duplicate results for model={_key[0]!r} task={_key[1]!r} " - f"n_shot={_key[2]!r} metric={_key[3]!r}: " - f"{_deduped[_key].get('performance')} superseded by " - f"{_row.get('performance')} (newest result file wins)" + _rank = (_row.get("limit") is None, _row.get("evaluated_at") or "", _i) + if _key not in _deduped: + _deduped[_key] = (_rank, _row) + continue + _old_rank, _old = _deduped[_key] + _kept, _dropped = (_row, _old) if _rank > _old_rank else (_old, _row) + if _kept.get("performance") != _dropped.get("performance"): + _where = ( + f"model={_key[0]!r} task={_key[1]!r} n_shot={_key[2]!r} " + f"metric={_key[3]!r}" ) - _deduped[_key] = _row - rows = list(_deduped.values()) + if _kept.get("limit") is None and _dropped.get("limit") is not None: + logging.warning( + f"Kept the full evaluation for {_where} " + f"({_kept.get('performance')}); ignored a --limit " + f"{_dropped.get('limit')} test result " + f"({_dropped.get('performance')})" + ) + else: + logging.warning( + f"Duplicate results for {_where}: " + f"{_dropped.get('performance')} superseded by " + f"{_kept.get('performance')} (newest evaluation wins)" + ) + if _kept is _row: + _deduped[_key] = (_rank, _row) + rows = [_row for _, _row in _deduped.values()] + + limited = [r for r in rows if r.get("limit") is not None] + if limited: + logging.warning( + f"{len(limited)} of {len(rows)} results come from --limit test " + f"runs, not full evaluations (marked in the outputs)" + ) + + run_provenance = [ + {**provenance_cache[_dir], "_path": str(_dir / "provenance.json")} + for _dir in sorted({row_run[id(r)] for r in rows if id(r) in row_run}) + ] df = pd.DataFrame(rows) df.to_csv(output_csv, index=False) @@ -728,18 +862,22 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None for _, job in jobs_df.iterrows(): job_tuple = (job["model_path"], job["task_path"], job["n_shot"]) + needs_full = bool(job.get("_needs_full")) is_completed = False - if job_tuple in completed_jobs: + if job_tuple in completed_jobs and ( + completed_jobs[job_tuple] or not needs_full + ): is_completed = True else: - for completed_job in completed_jobs: + for completed_job, has_full in completed_jobs.items(): completed_model, completed_task, completed_n_shot = completed_job if ( job["n_shot"] == completed_n_shot and job["task_path"] == completed_task + and (has_full or not needs_full) and _model_paths_match( str(job["model_path"]), str(completed_model) ) @@ -757,7 +895,7 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None logging.info(f"Missing jobs: {len(missing_jobs)}") if len(missing_jobs) > 0: - missing_df = pd.DataFrame(missing_jobs) + missing_df = pd.DataFrame(missing_jobs).drop(columns=["_needs_full"]) _out = Path(output_csv) missing_csv = str( _out.with_name(f"{_out.stem}_missing{_out.suffix or '.csv'}") @@ -782,7 +920,7 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None # Structured output: versioned JSON and Markdown report # --------------------------------------------------------------------------- -SCHEMA_VERSION = "1.2" +SCHEMA_VERSION = "1.3" def _collector_git_commit() -> str | None: @@ -816,7 +954,9 @@ def write_results_json( overrides, or null). Schema v1.2 adds `oellm_version`, `collector_git_commit`, per-run provenance under `runs`, and a reserved extensible `metadata` namespace (future additive fields — e.g. safety / - compliance metadata — land there without a schema migration). + compliance metadata — land there without a schema migration). Schema v1.3 + adds per-row `run`, `limit`, `quantization` and `evaluated_at`, and `runs` + lists only the runs those rows came from. """ output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) @@ -831,6 +971,10 @@ def write_results_json( "metric": row.get("metric_name", ""), "performance": row.get("performance", 0.0), "performance_normalized": row.get("performance_normalized"), + "limit": row.get("limit"), + "quantization": row.get("quantization"), + "evaluated_at": row.get("evaluated_at"), + "run": row.get("run"), } ) @@ -863,6 +1007,7 @@ def write_results_markdown( ] has_raw_fallback = False has_lower_is_better = False + has_limited = False for row in rows: model = row.get("model_name", "") task = row.get("task", "") @@ -875,6 +1020,9 @@ def write_results_markdown( raw = row.get("performance", 0.0) perf_cell = f"{raw:.4f}*" has_raw_fallback = True + if row.get("limit") is not None: + perf_cell += f" † (limit {row['limit']})" + has_limited = True if any(k in metric.lower() for k in ("wer", "mer", "cer")): has_lower_is_better = True lines.append(f"| {model} | {task} | {n_shot} | {metric} | {perf_cell} |") @@ -883,6 +1031,8 @@ def write_results_markdown( footnotes = [] if has_raw_fallback: footnotes.append("> `*` = raw value (metric scale not in `METRIC_NATIVE_SCALE`).") + if has_limited: + footnotes.append("> `†` = test run with --limit, not a full evaluation.") if has_lower_is_better: footnotes.append("> WER/MER/CER are lower-is-better.") if footnotes: diff --git a/oellm/scheduler.py b/oellm/scheduler.py index f09de36d..10547b10 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -156,6 +156,20 @@ def _probe_engine_versions(venv_path: str | None) -> dict[str, str]: return versions +def _cluster_setting_names() -> set[str]: + """Names of all settings in clusters.yaml.""" + import yaml + + clusters = yaml.safe_load((files("oellm.resources") / "clusters.yaml").read_text()) + return { + key + for section in (clusters or {}).values() + if isinstance(section, dict) + for key in section + if key != "hostname_pattern" + } + + @capture_third_party_output_from_kwarg("verbose") def schedule_evals( models: str | None = None, @@ -197,7 +211,8 @@ def schedule_evals( tasks: A string of comma-separated task names (lm_eval) or paths. Requires `n_shot` to be provided. Tasks here are assumed to be lm_eval unless otherwise handled via CSV. task_groups: A string of comma-separated task group names defined in `task-groups.yaml`. - Each group expands into concrete (task, n_shots, suite) entries; `n_shot` is ignored for groups. + Each group expands into concrete (task, n_shots, suite) entries and sets its own shots; + `n_shot` applies to `tasks` only. Given together, `tasks` and `task_groups` are both scheduled. n_shot: An integer or list of integers specifying the number of shots applied to `tasks`. eval_csv_path: A path to a CSV file containing evaluation data. Warning: exclusive argument. Cannot specify `models`, `tasks`, `task_groups`, or `n_shot` when `eval_csv_path` is provided. @@ -303,12 +318,34 @@ def schedule_evals( ) elif models: - if group_names is None: + if tasks and not n_shot: + raise ValueError("n_shot is required when specifying individual tasks.") + if n_shot and group_names is not None and not tasks: + raise ValueError( + "n_shot applies to tasks only; task groups set their own shots " + "(see oellm-eval list-tasks)." + ) + if group_names is not None: + expanded = _expand_task_groups(group_names) + eval_jobs.extend( + [ + EvaluationJob( + model_path=model, + task_path=result.task, + n_shot=result.n_shot, + eval_suite=result.suite, + ) + for model in models + for result in expanded + ] + ) + if tasks: # Look up each bare task name in the registered groups so # ``--tasks belebele_eng_Latn_cf`` (lighteval) or ``--tasks # regiondial_refcocog_all`` (contrib) get routed correctly. # Tasks not in any group default to lm_eval. task_suite_map = _build_task_suite_map() + scheduled = {(j.model_path, j.task_path, j.n_shot) for j in eval_jobs} eval_jobs.extend( [ EvaluationJob( @@ -320,20 +357,7 @@ def schedule_evals( for model in models for task in tasks for shot in n_shot - ] - ) - else: - expanded = _expand_task_groups(group_names) - eval_jobs.extend( - [ - EvaluationJob( - model_path=model, - task_path=result.task, - n_shot=result.n_shot, - eval_suite=result.suite, - ) - for model in models - for result in expanded + if (model, task, shot) not in scheduled ] ) @@ -487,9 +511,17 @@ def _lower_suite_only(s: str) -> str: # Ensure that all datasets required by the tasks are cached locally to avoid # network access on compute nodes. if not skip_checks: + # Extra --tasks next to groups need their data too. + extra_tasks = sorted(set(tasks or [])) if group_names else [] dataset_specs = [] if group_names: dataset_specs = _collect_dataset_specs(group_names) + staged = {(spec.repo_id, spec.subset) for spec in dataset_specs} + dataset_specs += [ + spec + for spec in _lookup_dataset_specs_for_tasks(extra_tasks) + if (spec.repo_id, spec.subset) not in staged + ] else: # Look up individual tasks in task groups registry all_tasks = df["task_path"].unique().tolist() @@ -511,6 +543,16 @@ def _lower_suite_only(s: str) -> str: if group_names: hf_model_repos = _collect_hf_model_repos(group_names) hf_dataset_files = _collect_hf_dataset_files(group_names) + hf_model_repos += [ + repo + for repo in _lookup_hf_model_repos_for_tasks(extra_tasks) + if repo not in hf_model_repos + ] + hf_dataset_files += [ + spec + for spec in _lookup_hf_dataset_files_for_tasks(extra_tasks) + if spec not in hf_dataset_files + ] else: _all_task_names = df["task_path"].unique().tolist() hf_model_repos = _lookup_hf_model_repos_for_tasks(_all_task_names) @@ -580,6 +622,7 @@ def _lower_suite_only(s: str) -> str: ) # Apply slurm_template_var overrides (JSON object) + template_var_names: set[str] = set() if slurm_template_var: try: opts = json.loads(slurm_template_var) @@ -598,6 +641,7 @@ def _lower_suite_only(s: str) -> str: logging.info(f"Using time limit override: {time_limit}") else: os.environ[key] = str(value) + template_var_names.add(key) logging.info(f"Using slurm_template_var override: {key}={value}") if nodelist: @@ -698,12 +742,14 @@ def _lower_suite_only(s: str) -> str: if not os.environ.get("NODELIST"): sbatch_script = sbatch_script.replace("#SBATCH --nodelist=$NODELIST\n", "") - # Substitute $ENV_VAR occurrences from the environment — EXCLUDING SLURM_* - # runtime variables: when scheduling from inside an allocation - # (salloc/srun) those are set at render time and would be baked into the - # script (e.g. JOB_HOME losing its per-job uniqueness) instead of - # expanding on the compute node. - _template_env = {k: v for k, v in os.environ.items() if not k.startswith("SLURM_")} + # Fill in only cluster settings from the environment; the job's own + # variables (VENV_PATH, LIMIT, …) and SLURM_* values expand on the node. + render_names = _cluster_setting_names() | {"NODELIST"} | template_var_names + _template_env = { + k: v + for k, v in os.environ.items() + if k in render_names and not k.startswith("SLURM_") + } sbatch_script = Template(sbatch_script).safe_substitute(_template_env) sbatch_script_path = evals_dir / "submit_evals.sbatch" diff --git a/oellm/task_groups.py b/oellm/task_groups.py index f6a26486..932bf230 100644 --- a/oellm/task_groups.py +++ b/oellm/task_groups.py @@ -576,7 +576,7 @@ def _select_tasks(group_names: Iterable[str]) -> list[tuple[str, _Task]]: raise ValueError(f"Unknown task group(s): {', '.join(sorted(missing))}") selected: list[tuple[str, _Task]] = [] - seen: set[tuple[str, str]] = set() + seen: dict[tuple[str, str], int] = {} for name, filt in specs: group_pairs = list(_iter_group_tasks({name: parsed[name]})) if filt is None: @@ -598,12 +598,20 @@ def _select_tasks(group_names: Iterable[str]) -> list[tuple[str, _Task]]: ", ".join(lang for lang in filt if lang in matched), ) # De-duplicate tasks shared by several groups (e.g. the `all` super_group - # spans groups whose benchmarks overlap), so they are scheduled once. + # spans groups whose benchmarks overlap), so they are scheduled once, + # with every group's shots. for suite, t in kept: key = (suite, t.name) if key not in seen: - seen.add(key) + seen[key] = len(selected) selected.append((suite, t)) + continue + first = selected[seen[key]][1] + extra = [s for s in (t.n_shots or []) if s not in (first.n_shots or [])] + if extra: + merged = copy.copy(first) + merged.n_shots = [*(first.n_shots or []), *extra] + selected[seen[key]] = (suite, merged) return selected diff --git a/tests/test_audiobench.py b/tests/test_audiobench.py index d7617eb7..f1be6c22 100644 --- a/tests/test_audiobench.py +++ b/tests/test_audiobench.py @@ -326,6 +326,38 @@ def test_detect_model_flags_qwen2_audio(self, suite): def test_detect_model_flags_unknown_returns_none(self, suite): assert suite.detect_model_flags("some/unknown-model") is None + @pytest.mark.parametrize( + "path, key", + [ + ("/ckpts/qwen2-audio-7b-instruct-sft-step500", "Qwen2-Audio-7B-Instruct"), + ("my-org/Qwen2-Audio-7B-Instruct-finetuned", "Qwen2-Audio-7B-Instruct"), + ("tsinghua/SALMONN-7B", "SALMONN_7B"), # stock model, stock-only family + ], + ) + def test_detect_model_flags_accepts_checkpoints_and_stock_models( + self, suite, path, key + ): + assert suite.detect_model_flags(path) == key + + def test_detect_model_flags_refuses_checkpoints_of_stock_only_families(self, suite): + with pytest.raises(ValueError, match="only runs the stock model"): + suite.detect_model_flags("/ckpts/seallms-audio-7b-sft") + + @pytest.mark.parametrize( + "architecture, key", + [ + ("Qwen2AudioForConditionalGeneration", "Qwen2-Audio-7B-Instruct"), + ("LlamaForCausalLM", None), + ], + ) + def test_detect_model_flags_reads_the_family_from_config( + self, suite, tmp_path, architecture, key + ): + (tmp_path / "config.json").write_text( + json.dumps({"architectures": [architecture]}) + ) + assert suite.detect_model_flags(str(tmp_path)) == key + def test_parse_results_recognises_audiobench_json(self, suite): data = { "model_name_or_path": "/path/to/model", @@ -514,12 +546,10 @@ def _fake_audiobench_tree(self, tmp_path: Path) -> Path: @staticmethod def _score_file_path( - ab_dir: Path, model_name: str, dataset: str, metric: str + log_dir: Path, model_name: str, dataset: str, metric: str ) -> Path: """Mirror suite._extract_metrics' path construction.""" - return ( - ab_dir / "log_for_all_models" / model_name / f"{dataset}_{metric}_score.json" - ) + return log_dir / model_name / f"{dataset}_{metric}_score.json" def _fake_run_writing_score( self, ab_dir: Path, *, score_value: float, body_shape: str = "flat" @@ -529,10 +559,11 @@ def _fake_run_writing_score( """ def fake_run(cmd, cwd, env, check): - model_name = cmd[cmd.index("--model_name") + 1] - dataset = cmd[cmd.index("--dataset_name") + 1] + model_name = cmd[cmd.index("--model-name") + 1] + dataset = cmd[cmd.index("--dataset-name") + 1] metric = cmd[cmd.index("--metrics") + 1] - score_file = self._score_file_path(Path(cwd), model_name, dataset, metric) + log_dir = Path(cmd[cmd.index("--log-dir") + 1]) + score_file = self._score_file_path(log_dir, model_name, dataset, metric) score_file.parent.mkdir(parents=True, exist_ok=True) if body_shape == "flat": score_file.write_text(json.dumps({metric: score_value})) @@ -596,6 +627,24 @@ def test_run_unmapped_model_raises(self, tmp_path): env={"AUDIOBENCH_DIR": str(ab_dir)}, ) + def test_run_refuses_a_checkpoint_of_a_stock_only_family(self, tmp_path, monkeypatch): + """A CSV row can carry "audiobench:" for any path.""" + from oellm.contrib.audiobench import suite + + ab_dir = self._fake_audiobench_tree(tmp_path) + started = [] + monkeypatch.setattr(suite.subprocess, "run", lambda *a, **k: started.append(a)) + with pytest.raises(RuntimeError, match="only runs the stock model"): + suite.run( + model_path="/ckpts/seallms-audio-7b-sft", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags="seallms_audio_7b", + env={"AUDIOBENCH_DIR": str(ab_dir)}, + ) + assert started == [] + def test_run_invokes_subprocess_with_expected_cli(self, tmp_path): from oellm.contrib.audiobench import suite @@ -617,24 +666,16 @@ def test_run_invokes_subprocess_with_expected_cli(self, tmp_path): assert mock_sp.call_count == 1 cmd = mock_sp.call_args.args[0] - assert cmd[:2] == ["python", "src/main_evaluate.py"] + assert cmd[0] == sys.executable + assert cmd[1].endswith("oellm/contrib/audiobench/launch.py") - # AudioBench's actual main() signature: dataset_name / model_name - # / metrics / overwrite / number_of_samples. No --model, no - # --log_dir, no --data_dir. - assert cmd[cmd.index("--dataset_name") + 1] == "librispeech_test_clean" - assert cmd[cmd.index("--model_name") + 1] == "Qwen2-Audio-7B-Instruct" + assert cmd[cmd.index("--audiobench-dir") + 1] == str(ab_dir) + assert cmd[cmd.index("--dataset-name") + 1] == "librispeech_test_clean" + assert cmd[cmd.index("--model-name") + 1] == "Qwen2-Audio-7B-Instruct" assert cmd[cmd.index("--metrics") + 1] == "wer" - assert cmd[cmd.index("--overwrite") + 1] == "True" - assert cmd[cmd.index("--number_of_samples") + 1] == "100" - - # Flags AudioBench does NOT accept must not be in the cmd. - assert "--model" not in cmd # only --model_name exists upstream - assert "--log_dir" not in cmd # AudioBench writes to a fixed path - assert "--data_dir" not in cmd # split selection is via dataset_name + assert cmd[cmd.index("--number-of-samples") + 1] == "100" + assert "--checkpoint" not in cmd - # cwd is AUDIOBENCH_DIR so AudioBench's relative writes - # (log_for_all_models/...) land inside the clone. assert mock_sp.call_args.kwargs["cwd"] == str(ab_dir) # Output JSON is lmms-eval-shaped. @@ -668,7 +709,7 @@ def test_run_uses_per_split_dataset_name_for_gigaspeech2(self, tmp_path): ) cmd = mock_sp.call_args.args[0] - assert cmd[cmd.index("--dataset_name") + 1] == "gigaspeech2_thai" + assert cmd[cmd.index("--dataset-name") + 1] == "gigaspeech2_thai" assert "--data_dir" not in cmd def test_run_omits_number_of_samples_when_limit_empty(self, tmp_path): @@ -691,33 +732,7 @@ def test_run_omits_number_of_samples_when_limit_empty(self, tmp_path): ) cmd = mock_sp.call_args.args[0] - assert "--number_of_samples" not in cmd - - def test_run_always_passes_overwrite_true(self, tmp_path): - """AudioBench skips evaluation when a stale score file already - exists unless ``--overwrite True`` is passed; we always pass it - because we do our own deduplication via output_path. - """ - from oellm.contrib.audiobench import suite - - ab_dir = self._fake_audiobench_tree(tmp_path) - output_path = tmp_path / "result.json" - - with patch( - "oellm.contrib.audiobench.suite.subprocess.run", - side_effect=self._fake_run_writing_score(ab_dir, score_value=0.1), - ) as mock_sp: - suite.run( - model_path="Qwen/Qwen2-Audio-7B-Instruct", - task="audiobench_librispeech_test_clean", - n_shot=0, - output_path=output_path, - model_flags="Qwen2-Audio-7B-Instruct", - env={"AUDIOBENCH_DIR": str(ab_dir)}, - ) - - cmd = mock_sp.call_args.args[0] - assert cmd[cmd.index("--overwrite") + 1] == "True" + assert "--number-of-samples" not in cmd def test_run_nonzero_exit_raises(self, tmp_path): from oellm.contrib.audiobench import suite @@ -820,6 +835,73 @@ def test_run_score_file_without_metric_key_raises(self, tmp_path): ) +FAKE_MAIN = """ +import json, os +from model import Model +file_save_folder = "log_for_all_models" +def main(dataset_name, model_name, metrics, overwrite, number_of_samples): + model = Model(model_name) + os.makedirs(f"{file_save_folder}/{model_name}", exist_ok=True) + body = {metrics: 0.5, "loaded": model.loaded, "samples": number_of_samples, + "cwd": os.getcwd()} + for path in (f"{file_save_folder}/{model_name}/{dataset_name}_{metrics}_score.json", + os.environ["RECORD"]): + json.dump(body, open(path, "w")) +""" +FAKE_MODEL = """ +import importlib +class Model: + def __init__(self, name): + module = importlib.import_module("model_src.qwen2_audio_7b_instruct") + module.qwen2_audio_7b_instruct_model_loader(self) +""" +FAKE_LOADER = """ +model_path = "Qwen/Qwen2-Audio-7B-Instruct" +def qwen2_audio_7b_instruct_model_loader(self): + self.loaded = model_path +""" + + +def test_suite_run_evaluates_the_checkpoint(tmp_path): + """Real launch.py against a minimal AudioBench tree whose loader records its weights.""" + from oellm.contrib.audiobench import suite + + ab_dir = tmp_path / "AudioBench" + (ab_dir / "src" / "model_src").mkdir(parents=True) + (ab_dir / "src" / "main_evaluate.py").write_text(FAKE_MAIN) + (ab_dir / "src" / "model.py").write_text(FAKE_MODEL) + (ab_dir / "src" / "model_src" / "qwen2_audio_7b_instruct.py").write_text(FAKE_LOADER) + stale = ( + ab_dir + / "log_for_all_models/Qwen2-Audio-7B-Instruct/librispeech_test_clean_wer_score.json" + ) + stale.parent.mkdir(parents=True) + stale.write_text('{"wer": 0.42}') + record = tmp_path / "record.json" + + suite.run( + model_path="/ckpts/qwen2-audio-7b-instruct-sft", + task="audiobench_librispeech_test_clean", + n_shot=0, + output_path=tmp_path / "out.json", + model_flags="Qwen2-Audio-7B-Instruct", + env={ + **os.environ, + "AUDIOBENCH_DIR": str(ab_dir), + "RECORD": str(record), + "LIMIT": "3", + }, + ) + + seen = json.loads(record.read_text()) + assert seen["loaded"] == "/ckpts/qwen2-audio-7b-instruct-sft" + assert (seen["samples"], seen["cwd"]) == (3, str(ab_dir.resolve())) + assert stale.read_text() == '{"wer": 0.42}' # the clone's files are untouched + body = json.loads((tmp_path / "out.json").read_text()) + assert body["model_name_or_path"] == "/ckpts/qwen2-audio-7b-instruct-sft" + assert body["results"]["audiobench_librispeech_test_clean"]["wer"] == 0.5 + + class _FakeCompletedProcess: """Stand-in for subprocess.CompletedProcess.""" diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 26ed5b15..02315efc 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -1,6 +1,7 @@ """Tests for collect_results() covering both lm-eval and lmms-eval output formats.""" import json +import os from pathlib import Path import pandas as pd @@ -438,7 +439,7 @@ def test_json_file_written_alongside_csv(self, tmp_path): json_path = tmp_path / "out.json" assert json_path.exists() envelope = json.loads(json_path.read_text()) - assert envelope["version"] == "1.2" + assert envelope["version"] == "1.3" assert len(envelope["results"]) == 1 record = envelope["results"][0] assert record["task"] == "copa" @@ -621,3 +622,156 @@ def test_missing_csv_name_without_suffix(self, tmp_path): (tmp_path / "outfile").read_text() != (tmp_path / "outfile_missing.csv").read_text() ) + + +# ── --limit test runs, run details, counters ───────────────────────────────── + +JOBS_CSV = ( + "model_path,task_path,n_shot,eval_suite\nEleutherAI/pythia-70m,copa,0,lm_eval\n" +) + + +def _lm_eval_result(task, acc, limit=None, date=None, model_args=""): + return { + "model_name": "EleutherAI/pythia-70m", + "results": {task: {"alias": task, "sample_len": 100, "acc,none": acc}}, + "n-shot": {task: 0}, + "config": {"limit": limit, "model_args": model_args}, + **({"date": date} if date else {}), + } + + +def _make_run(root, name, limit, payloads=(), jobs=False, provenance=True): + run = root / name + (run / "results").mkdir(parents=True) + if provenance: + (run / "provenance.json").write_text(json.dumps({"limit": limit})) + for i, payload in enumerate(payloads): + write_result(run / "results", payload, f"r{i}.json") + if jobs: + (run / "jobs.csv").write_text(JOBS_CSV) + return run + + +def _envelope(tmp_path): + return json.loads((tmp_path / "out.json").read_text()) + + +class TestLimitedRuns: + def test_full_evaluation_beats_newer_limited_run(self, tmp_path): + runs = tmp_path / "runs" + _make_run(runs, "full", None, [_lm_eval_result("copa", 0.57, date=1.7e9)]) + _make_run(runs, "smoke", 4, [_lm_eval_result("copa", 0.5, 4, 1.8e9)]) + + collect_results(str(runs), output_csv=str(tmp_path / "out.csv")) + + env = _envelope(tmp_path) + assert [(r["performance"], r["limit"], r["run"]) for r in env["results"]] == [ + (0.57, None, "full") + ] + assert [r["limit"] for r in env["runs"]] == [None] # only runs with rows + + def test_limited_only_result_is_kept_and_marked(self, tmp_path): + _make_run(tmp_path / "runs", "smoke", 4, [_lm_eval_result("copa", 0.5)]) + + collect_results(str(tmp_path / "runs"), output_csv=str(tmp_path / "out.csv")) + + assert _envelope(tmp_path)["results"][0]["limit"] == 4 + assert "† (limit 4)" in (tmp_path / "out.md").read_text() + + def test_limit_read_from_the_engine_without_provenance(self, tmp_path): + df = run_collect(tmp_path, _lm_eval_result("copa", 0.5, limit=16.0)) + assert df.iloc[0]["limit"] == 16 + + def test_newest_evaluation_wins_by_engine_date(self, tmp_path): + results = tmp_path / "results" + results.mkdir() + write_result(results, _lm_eval_result("copa", 0.7, date=1.8e9), "newer.json") + write_result(results, _lm_eval_result("copa", 0.6, date=1.7e9), "older.json") + os.utime(results / "newer.json", (1_000_000, 1_000_000)) # copied first + + collect_results(str(results), output_csv=str(tmp_path / "out.csv")) + + assert _envelope(tmp_path)["results"][0]["performance"] == 0.7 + + def test_check_needs_a_full_result_for_a_full_job(self, tmp_path): + runs = tmp_path / "runs" + _make_run(runs, "full", None, jobs=True) + _make_run(runs, "smoke", 4, [_lm_eval_result("copa", 0.5, 4)], jobs=True) + + collect_results(str(runs), output_csv=str(tmp_path / "out.csv"), check=True) + + missing = pd.read_csv(tmp_path / "out_missing.csv") + assert list(missing.columns) == [ + "model_path", + "task_path", + "n_shot", + "eval_suite", + ] + assert list(missing["task_path"]) == ["copa"] + + @pytest.mark.parametrize("provenance", [True, False]) + def test_check_accepts_a_limited_result_for_a_limited_or_old_job( + self, tmp_path, provenance + ): + result = _lm_eval_result("copa", 0.5, 4) + run = _make_run(tmp_path, "r", 4, [result], jobs=True, provenance=provenance) + + collect_results(str(run), output_csv=str(tmp_path / "out.csv"), check=True) + + assert not (tmp_path / "out_missing.csv").exists() + + +class TestRunDetails: + def test_quantization_comes_from_what_the_engine_loaded(self, tmp_path): + lm = _lm_eval_result("copa", 0.6, model_args="pretrained=m,load_in_4bit=True") + lighteval = { + "config_general": {"model_name": "EleutherAI/pythia-70m"}, + "results": {"belebele_eng_Latn_cf|0": {"acc_norm": 0.4}}, + } + run = _make_run(tmp_path / "runs", "q", None, [lm, lighteval]) + (run / "provenance.json").write_text(json.dumps({"quantization": "4bit"})) + + collect_results(str(tmp_path / "runs"), output_csv=str(tmp_path / "out.csv")) + + rows = {r["task"]: r["quantization"] for r in _envelope(tmp_path)["results"]} + assert rows == {"copa": "4bit", "belebele_eng_Latn_cf": None} + + def test_lmms_eval_dates_are_utc_plus_8(self, tmp_path): + data = { + "model_name": "m", + "results": {"realworldqa": {"exact_match,none": 0.5}}, + "configs": {"realworldqa": {"num_fewshot": 0}}, + "date": "20260922_165022", + } + run_collect(tmp_path, data) + assert ( + _envelope(tmp_path)["results"][0]["evaluated_at"] + == "2026-09-22T08:50:22+00:00" + ) + + @pytest.mark.parametrize("where", ["results", "."]) + def test_relative_paths_keep_the_run(self, tmp_path, monkeypatch, where): + run = _make_run(tmp_path, "smoke", 8, [_lm_eval_result("copa", 0.5)]) + monkeypatch.chdir(run / where) + + collect_results(".", output_csv=str(tmp_path / "out.csv")) + + row = _envelope(tmp_path)["results"][0] + assert (row["run"], row["limit"]) == ("smoke", 8) + + +class TestCounters: + def test_sample_len_is_not_the_score(self, tmp_path): + data = { + "model_name": "m", + "results": {"humaneval": {"sample_len": 164, "pass@1,create_test": 0.12}}, + "n-shot": {"humaneval": 0}, + } + row = run_collect(tmp_path, data).iloc[0] + assert (row["metric_name"], row["performance"]) == ("pass@1,create_test", 0.12) + + def test_a_result_with_only_counters_gives_no_row(self, tmp_path): + counters = {"sample_len": 10, "samples": 499, "gpt_eval_score,none": None} + data = {"model_name": "m", "results": {"x": counters}, "n-shot": {"x": 0}} + assert run_collect(tmp_path, data).empty diff --git a/tests/test_collection_and_scheduling.py b/tests/test_collection_and_scheduling.py index a218bd5d..88a15375 100644 --- a/tests/test_collection_and_scheduling.py +++ b/tests/test_collection_and_scheduling.py @@ -240,6 +240,17 @@ def test_voicebench_judge_is_likert_scale(self): 3.4, "llm_as_judge_eval,none", "voicebench_commoneval" ) == pytest.approx(68.0) + @pytest.mark.parametrize( + "task, raw, expected", + [ + ("alpaca_audio", 62.0, 62.0), # 0–100 + ("air_bench_chat_sound", 6.5, 65.0), # 1–10 + ("wavcaps", 3.1, 62.0), # 0–5, the gpt_eval default + ], + ) + def test_gpt_eval_judge_scales(self, task, raw, expected): + assert _normalize_to_100(raw, "gpt_eval,none", task) == pytest.approx(expected) + # ── Envelope v1.2 + provenance sidecar ─────────────────────────────────────── @@ -259,7 +270,7 @@ def test_sidecar_embedded_and_namespace_reserved(self, tmp_path): out = tmp_path / "out.csv" collect_results(str(tmp_path), str(out)) envelope = json.loads((tmp_path / "out.json").read_text()) - assert envelope["version"] == "1.2" + assert envelope["version"] == "1.3" assert envelope["metadata"] == {} assert envelope["oellm_version"] == __version__ assert envelope["runs"][0]["model_revisions"] == {"m": "abc123"} diff --git a/tests/test_image_task_groups.py b/tests/test_image_task_groups.py index 62c1f9e9..66dafb1e 100644 --- a/tests/test_image_task_groups.py +++ b/tests/test_image_task_groups.py @@ -31,7 +31,7 @@ EXPECTED_DATASETS = { "lmms-lab/VQAv2", "lmms-lab/MMBench", - "MMMU/MMMU", + "lmms-lab/MMMU", "lmms-lab/ChartQA", "lmms-lab/DocVQA", "lmms-lab/textvqa", @@ -99,7 +99,7 @@ def test_vqav2_dataset_included(self): def test_mmmu_dataset_included(self): specs = _collect_dataset_specs([IMAGE_TASK_GROUP]) repo_ids = {s.repo_id for s in specs} - assert "MMMU/MMMU" in repo_ids + assert "lmms-lab/MMMU" in repo_ids class TestImageTaskGroupScheduleEvals: diff --git a/tests/test_reporter.py b/tests/test_reporter.py index 81f08c23..69d78ad1 100644 --- a/tests/test_reporter.py +++ b/tests/test_reporter.py @@ -32,7 +32,7 @@ def test_write_json_schema_version(tmp_path: Path) -> None: out = tmp_path / "results.json" write_results_json(_SAMPLE_ROWS, out) data = json.loads(out.read_text()) - assert data["version"] == SCHEMA_VERSION == "1.2" + assert data["version"] == SCHEMA_VERSION == "1.3" def test_write_json_result_fields(tmp_path: Path) -> None: @@ -48,6 +48,10 @@ def test_write_json_result_fields(tmp_path: Path) -> None: "metric", "performance", "performance_normalized", + "limit", + "quantization", + "evaluated_at", + "run", } @@ -77,7 +81,7 @@ def test_write_json_empty_rows(tmp_path: Path) -> None: write_results_json([], out) data = json.loads(out.read_text()) assert data["results"] == [] - assert data["version"] == "1.2" + assert data["version"] == "1.3" def test_write_json_creates_parent_dirs(tmp_path: Path) -> None: diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index cd9381c5..38a37516 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -13,9 +13,9 @@ ALL_TASK_GROUPS = list(_config["task_groups"].keys()) -@pytest.mark.parametrize("n_shot", [None, 0]) @pytest.mark.parametrize("task_groups", ALL_TASK_GROUPS) -def test_schedule_evals(tmp_path, n_shot, task_groups): +def test_schedule_evals(tmp_path, task_groups): + # n_shot with a group is refused (test_schedule_inputs.py). with ( patch("oellm.scheduler._load_cluster_env"), patch("oellm.scheduler._num_jobs_in_queue", return_value=0), @@ -28,7 +28,6 @@ def test_schedule_evals(tmp_path, n_shot, task_groups): schedule_evals( models="EleutherAI/pythia-70m", task_groups=task_groups, - n_shot=n_shot, skip_checks=True, venv_path=str(Path(sys.prefix)), dry_run=True, diff --git a/tests/test_schedule_inputs.py b/tests/test_schedule_inputs.py new file mode 100644 index 00000000..26d5778b --- /dev/null +++ b/tests/test_schedule_inputs.py @@ -0,0 +1,112 @@ +"""Scheduling inputs: CLI flags, task groups and the login-shell environment.""" + +import csv +import os +import subprocess +from unittest.mock import patch + +import pytest + +from oellm.main import schedule_evals +from oellm.task_groups import _collect_dataset_specs, _expand_task_groups + + +def _schedule(tmp_path, env=None, **kw): + """Dry-run schedule; return the rendered job script and jobs.csv rows.""" + out = tmp_path / "out" + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(out), **(env or {})}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", dry_run=True, skip_checks=True, **kw + ) + run = next(out.iterdir()) + with open(run / "jobs.csv") as f: + rows = list(csv.DictReader(f)) + return (run / "submit_evals.sbatch").read_text(), rows + + +def test_exported_job_variables_do_not_override_flags(tmp_path): + env = {"VENV_PATH": "/stray/venv", "LIMIT": "100", "MODEL_DIR": "/stray/models"} + sbatch, _ = _schedule( + tmp_path, env=env, tasks="copa", n_shot=[0], venv_path="/my/venv", limit=5 + ) + assert "/stray" not in sbatch + assert 'VENV_PATH="/my/venv"' in sbatch + assert 'source "$VENV_PATH/bin/activate"' in sbatch + assert 'export LIMIT="5"' in sbatch + assert "${LIMIT:+--limit $LIMIT}" in sbatch + + +def test_tasks_and_groups_are_both_scheduled(tmp_path): + _, rows = _schedule( + tmp_path, tasks="gsm8k,hellaswag", n_shot=[10], task_groups="open-sci-0.01" + ) + tasks = [(r["task_path"], r["n_shot"]) for r in rows] + assert ("gsm8k", "10") in tasks + assert tasks.count(("hellaswag", "10")) == 1 # already in the group + assert len(rows) == len(_expand_task_groups(["open-sci-0.01"])) + 1 + + +def test_n_shot_with_groups_only_is_refused(tmp_path): + with pytest.raises(ValueError, match="n_shot applies to tasks only"): + _schedule(tmp_path, n_shot=[0], task_groups="open-sci-0.01") + + +def test_shared_tasks_keep_every_groups_shots(): + # hellaswag: 10-shot in open-sci-0.01, 0- and 10-shot in dclm-core-22. + both = _expand_task_groups(["open-sci-0.01", "dclm-core-22"]) + assert {r.n_shot for r in both if r.task == "hellaswag"} == {0, 10} + alone = _expand_task_groups(["open-sci-0.01"]) + assert {r.n_shot for r in alone if r.task == "hellaswag"} == {10} + + +def test_tasks_next_to_groups_get_their_data_staged(tmp_path, monkeypatch): + staged = [] + monkeypatch.setattr( + "oellm.envcheck.check_scheduled_environment", lambda *a, **k: None + ) + with ( + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch("oellm.scheduler._ensure_runtime_environment"), + patch("oellm.scheduler._process_model_paths", return_value={}), + patch("oellm.scheduler._probe_engine_versions", return_value={}), + patch( + "oellm.scheduler._pre_download_datasets_from_specs", + side_effect=lambda specs, **_: staged.extend(specs), + ), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path / "out")}), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="xcopa", + n_shot=[0], + task_groups="open-sci-0.01", + dry_run=True, + venv_path=str(tmp_path / "venv"), + ) + group_repos = {s.repo_id for s in _collect_dataset_specs(["open-sci-0.01"])} + assert {s.repo_id for s in staged} == group_repos | {"xcopa"} + + +def test_the_job_uses_the_cache_the_pre_download_filled(tmp_path): + sbatch, _ = _schedule(tmp_path, env={"HF_HOME": "/hf"}, tasks="copa", n_shot=[0]) + line = next( + ln for ln in sbatch.splitlines() if ln.startswith("export HF_DATASETS_CACHE=") + ) + for exported, expected in (("/scratch/ds", "/scratch/ds"), (None, "/hf/datasets")): + env = { + "PATH": os.environ["PATH"], + **({"HF_DATASETS_CACHE": exported} if exported else {}), + } + out = subprocess.run( + ["bash", "-c", f'{line}; echo "$HF_DATASETS_CACHE"'], + env=env, + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == expected diff --git a/tests/test_task_groups.py b/tests/test_task_groups.py index b49f4ffc..b98dffed 100644 --- a/tests/test_task_groups.py +++ b/tests/test_task_groups.py @@ -1,3 +1,5 @@ +import pytest + from oellm.task_groups import ( _expand_lang_templates, _expand_task_groups, @@ -182,3 +184,23 @@ def test_global_mmlu_expands_to_16_tasks(self): def test_global_piqa_completions_expands_to_32_tasks(self): results = _expand_task_groups(["global-piqa-eu-completions"]) assert len(results) == 32 + + +class TestPreDownloadMatchesWhatTheTaskLoads: + """Offline caches are keyed by dataset id, so specs must match the task's.""" + + @pytest.mark.parametrize( + "group, repo_id", + [ + # custom_lm_eval_tasks/arc_mt/arc_challenge_mt_is.yaml + ("arc-challenge-mt-eu[isl_Latn]", "mideind/icelandic-arc-challenge"), + # lm-eval 0.4.12 tasks/xcopa/default_et.yaml: dataset_path: xcopa + ("generic-multilingual", "xcopa"), + # lmms-eval 45c766f tasks/mmmu/mmmu_val.yaml: dataset_path: lmms-lab/MMMU + ("image-mmmu", "lmms-lab/MMMU"), + ], + ) + def test_spec_names_the_loaded_dataset(self, group, repo_id): + from oellm.task_groups import _collect_dataset_specs + + assert repo_id in {spec.repo_id for spec in _collect_dataset_specs([group])} From 0a3e3b205d1a4f8aa3bb8fabb583a2537c719d69 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Thu, 24 Sep 2026 16:05:03 +0200 Subject: [PATCH 39/44] [Base] Drop duplicate upstream tests, keep the cases ours missed --- tests/test_collect_results.py | 171 ------------------------ tests/test_collection_and_scheduling.py | 34 +++++ tests/test_hf_home_required.py | 74 ---------- tests/test_schedule_inputs.py | 16 +++ 4 files changed, 50 insertions(+), 245 deletions(-) delete mode 100644 tests/test_hf_home_required.py diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 3302e448..02315efc 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -1,7 +1,6 @@ """Tests for collect_results() covering both lm-eval and lmms-eval output formats.""" import json -import logging import os from pathlib import Path @@ -776,173 +775,3 @@ def test_a_result_with_only_counters_gives_no_row(self, tmp_path): counters = {"sample_len": 10, "samples": 499, "gpt_eval_score,none": None} data = {"model_name": "m", "results": {"x": counters}, "n-shot": {"x": 0}} assert run_collect(tmp_path, data).empty - - -# ── lm-eval group aggregates ───────────────────────────────────────────────── - - -@pytest.fixture(autouse=True) -def _restore_root_logger(): - # collect_results reconfigures the root logger (RichHandler at INFO) and - # would leave it that way for every test file that runs afterwards. - root = logging.getLogger() - handlers, level = root.handlers[:], root.level - yield - root.handlers[:] = handlers - root.setLevel(level) - - -MODEL = "/scratch/project_465002530/checkpoints/iter_0002000" -MMLU_SUBGROUPS = [ - "mmlu_humanities", - "mmlu_other", - "mmlu_social_sciences", - "mmlu_stem", -] - - -def _metrics(**values: float) -> dict: - """A result entry as lm-eval writes it, with its ``,none`` filter suffix.""" - entry: dict = {"alias": "task"} - for name, value in values.items(): - entry[f"{name},none"] = value - entry[f"{name}_stderr,none"] = 0.01 - return entry - - -def _write_case(tmp_path: Path, name: str, payload: dict, jobs: list[dict]) -> Path: - case_dir = tmp_path / name - (case_dir / "results").mkdir(parents=True) - pd.DataFrame(jobs).to_csv(case_dir / "jobs.csv", index=False) - (case_dir / "results" / "a1b2c3d4e5.json").write_text(json.dumps(payload)) - return case_dir - - -def _collect(case_dir: Path) -> pd.DataFrame: - output_csv = case_dir / "eval_results.csv" - collect_results(str(case_dir), str(output_csv), check=True) - if not output_csv.exists(): - return pd.DataFrame(columns=["model_name", "task", "n_shot", "performance"]) - return pd.read_csv(output_csv) - - -def _missing(case_dir: Path) -> pd.DataFrame: - missing_csv = case_dir / "eval_results_missing.csv" - if not missing_csv.exists(): - return pd.DataFrame(columns=["model_path", "task_path", "n_shot"]) - return pd.read_csv(missing_csv) - - -def _mmlu_case(tmp_path: Path) -> Path: - """`lm_eval --tasks mmlu`: subgroups aggregated before the parent.""" - results = {sub: _metrics(acc=0.30) for sub in MMLU_SUBGROUPS} - results["mmlu"] = _metrics(acc=0.4123) - results["mmlu_abstract_algebra"] = _metrics(acc=0.25) - - groups = {sub: results[sub] for sub in MMLU_SUBGROUPS} - groups["mmlu"] = results["mmlu"] - - payload = { - "model_name": MODEL, - "results": results, - "groups": groups, - "group_subtasks": { - **{sub: ["mmlu_abstract_algebra"] for sub in MMLU_SUBGROUPS}, - "mmlu": MMLU_SUBGROUPS, - }, - "n-shot": {"mmlu_abstract_algebra": 5}, - } - jobs = [{"model_path": MODEL, "task_path": "mmlu", "n_shot": 5}] - return _write_case(tmp_path, "mmlu", payload, jobs) - - -def test_mmlu_reports_the_aggregate_not_a_subcategory(tmp_path: Path) -> None: - df = _collect(_mmlu_case(tmp_path)) - - assert list(df["task"]) == ["mmlu"] - assert df.loc[0, "performance"] == 0.4123 - - -def test_mmlu_is_not_reported_as_missing(tmp_path: Path) -> None: - case_dir = _mmlu_case(tmp_path) - _collect(case_dir) - - assert _missing(case_dir).empty - - -def test_every_top_level_group_is_collected(tmp_path: Path) -> None: - """Two independent groups in one file: neither may be dropped.""" - results = { - "arc_challenge": _metrics(acc=0.50, acc_norm=0.55), - "hellaswag": _metrics(acc=0.60, acc_norm=0.66), - } - payload = { - "model_name": MODEL, - "results": results, - "groups": dict(results), - "group_subtasks": {"arc_challenge": [], "hellaswag": []}, - "n-shot": {"arc_challenge": 0, "hellaswag": 0}, - } - jobs = [ - {"model_path": MODEL, "task_path": "arc_challenge", "n_shot": 0}, - {"model_path": MODEL, "task_path": "hellaswag", "n_shot": 0}, - ] - case_dir = _write_case(tmp_path, "two_groups", payload, jobs) - - df = _collect(case_dir) - - assert dict(zip(df["task"], df["performance"], strict=True)) == { - "arc_challenge": 0.55, - "hellaswag": 0.66, - } - assert _missing(case_dir).empty - - -def test_a_plain_task_beside_a_group_survives(tmp_path: Path) -> None: - """The per-task loop must still run when the file also holds a group.""" - payload = { - "model_name": MODEL, - "results": { - "mmlu": _metrics(acc=0.41), - "mmlu_humanities": _metrics(acc=0.30), - "winogrande": _metrics(acc=0.72), - }, - "groups": { - "mmlu_humanities": _metrics(acc=0.30), - "mmlu": _metrics(acc=0.41), - }, - "group_subtasks": {"mmlu": ["mmlu_humanities"], "mmlu_humanities": []}, - "n-shot": {"mmlu": 5, "mmlu_humanities": 5, "winogrande": 5}, - } - jobs = [ - {"model_path": MODEL, "task_path": "mmlu", "n_shot": 5}, - {"model_path": MODEL, "task_path": "winogrande", "n_shot": 5}, - ] - case_dir = _write_case(tmp_path, "group_and_task", payload, jobs) - - df = _collect(case_dir) - - assert dict(zip(df["task"], df["performance"], strict=True)) == { - "mmlu": 0.41, - "winogrande": 0.72, - } - assert _missing(case_dir).empty - - -def test_results_without_groups_are_unaffected(tmp_path: Path) -> None: - """Control: a lighteval-style file carries no `groups` map.""" - payload = { - "config_general": {"model_name": MODEL}, - "results": { - "belebele_eus_Latn_cf|0": {"acc_norm": 0.31}, - "all": {"acc_norm": 0.31}, - }, - } - jobs = [{"model_path": MODEL, "task_path": "belebele_eus_Latn_cf", "n_shot": 0}] - case_dir = _write_case(tmp_path, "lighteval", payload, jobs) - - df = _collect(case_dir) - - assert list(df["task"]) == ["belebele_eus_Latn_cf"] - assert df.loc[0, "performance"] == 0.31 - assert _missing(case_dir).empty diff --git a/tests/test_collection_and_scheduling.py b/tests/test_collection_and_scheduling.py index 88a15375..a03ee9bb 100644 --- a/tests/test_collection_and_scheduling.py +++ b/tests/test_collection_and_scheduling.py @@ -127,6 +127,40 @@ def test_standalone_task_in_group_file_is_kept(self, tmp_path): ) assert {r["task"] for r in rows} == {"g1", "lone"} + def test_subgroup_stays_inside_its_parent(self, tmp_path): + # lm-eval lists a subgroup before its parent and gives shots only on leaf tasks. + sub, parent = {"acc,none": 0.3}, {"acc,none": 0.41} + d = tmp_path / "results" + d.mkdir() + _write( + d, + "r.json", + { + **M, + "results": { + "mmlu_humanities": sub, + "mmlu": parent, + "mmlu_astronomy": {"acc,none": 0.25}, + }, + "groups": {"mmlu_humanities": sub, "mmlu": parent}, + "group_subtasks": { + "mmlu_humanities": ["mmlu_astronomy"], + "mmlu": ["mmlu_humanities"], + }, + "n-shot": {"mmlu_astronomy": 5}, + }, + ) + (tmp_path / "jobs.csv").write_text( + "model_path,task_path,n_shot,eval_suite\nm,mmlu,5,lm_eval\n" + ) + out = tmp_path / "out.csv" + collect_results(str(tmp_path), str(out), check=True) + rows = [ + (r["task"], r["n_shot"], float(r["performance"])) for r in _rows(str(out)) + ] + assert rows == [("mmlu", "5", 0.41)] + assert not out.with_name("out_missing.csv").exists() + def test_mmlu_pro_is_not_eaten_by_prefix_rule(self, tmp_path): rows = self._collect( tmp_path, diff --git a/tests/test_hf_home_required.py b/tests/test_hf_home_required.py deleted file mode 100644 index 54060fef..00000000 --- a/tests/test_hf_home_required.py +++ /dev/null @@ -1,74 +0,0 @@ -"""HF_HOME must be set before a cluster run, and must not block --local.""" - -import os -from pathlib import Path -from unittest.mock import patch - -import pytest - -from oellm.main import schedule_evals - - -def _schedule(tmp_path, **kwargs): - return schedule_evals( - models="EleutherAI/pythia-70m", - tasks="hellaswag", - n_shot=0, - skip_checks=True, - venv_path=str(tmp_path / "venv"), - dry_run=True, - **kwargs, - ) - - -def _cluster(monkeypatch, tmp_path, **settings): - """Run the real _load_cluster_env (where ELLIOT checks HF_HOME) on a synthetic cluster.""" - cluster = { - "hostname_pattern": "*", - "PARTITION": "gpu", - "EVAL_BASE_DIR": str(tmp_path), - "EVAL_OUTPUT_DIR": str(tmp_path), - "GPUS_PER_NODE": 1, - **settings, - } - monkeypatch.setattr("oellm.utils.yaml.safe_load", lambda *_a, **_k: {"test": cluster}) - monkeypatch.delenv("HF_HOME", raising=False) - - -def test_missing_hf_home_is_rejected(tmp_path, monkeypatch): - """Unset, HF_HOME renders an empty bind path and every job dies in singularity.""" - _cluster(monkeypatch, tmp_path) - with ( - patch("oellm.scheduler._num_jobs_in_queue", return_value=0), - patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), - pytest.raises(RuntimeError, match="HF_HOME"), - ): - _schedule(tmp_path) - - -def test_hf_home_from_clusters_yaml_is_accepted(tmp_path, monkeypatch): - """Clusters that declare HF_HOME (e.g. jupiter) set it in _load_cluster_env.""" - _cluster(monkeypatch, tmp_path, HF_HOME=str(tmp_path / "cache")) - with ( - patch("oellm.scheduler._num_jobs_in_queue", return_value=0), - patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), - ): - _schedule(tmp_path) - - assert list(tmp_path.glob("**/submit_evals.sbatch")) - - -def test_local_runs_still_default_hf_home(tmp_path, monkeypatch): - """--local falls back to ~/.cache/huggingface and must not hit the guard.""" - monkeypatch.delenv("HF_HOME", raising=False) - # --local uses setdefault for these, so a value inherited from the caller's - # shell would win over the local default. - monkeypatch.delenv("EVAL_OUTPUT_DIR", raising=False) - monkeypatch.delenv("EVAL_BASE_DIR", raising=False) - monkeypatch.chdir(tmp_path) - with ( - patch("oellm.scheduler._num_jobs_in_queue", return_value=0), - patch.dict(os.environ), - ): - _schedule(tmp_path, local=True) - assert os.environ["HF_HOME"] == str(Path.home() / ".cache" / "huggingface") diff --git a/tests/test_schedule_inputs.py b/tests/test_schedule_inputs.py index 26d5778b..91fc5b9f 100644 --- a/tests/test_schedule_inputs.py +++ b/tests/test_schedule_inputs.py @@ -3,6 +3,7 @@ import csv import os import subprocess +from pathlib import Path from unittest.mock import patch import pytest @@ -110,3 +111,18 @@ def test_the_job_uses_the_cache_the_pre_download_filled(tmp_path): check=True, ) assert out.stdout.strip() == expected + + +def test_local_run_defaults_hf_home(tmp_path, monkeypatch): + monkeypatch.delenv("HF_HOME", raising=False) + with patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}): + schedule_evals( + models="EleutherAI/pythia-70m", + tasks="copa", + n_shot=[0], + local=True, + venv_path=str(tmp_path / "venv"), + dry_run=True, + skip_checks=True, + ) + assert os.environ["HF_HOME"] == str(Path.home() / ".cache" / "huggingface") From 098d8ca1e61c54dd703fc48029ab7645ad7e50dc Mon Sep 17 00:00:00 2001 From: islobozhan Date: Fri, 25 Sep 2026 15:04:49 +0200 Subject: [PATCH 40/44] [Base] Push collected results to the dashboard from the login node (oellm-eval push, collect --push) Push collected results to the dashboard from the login node (oellm-eval push, collect --push) --- README.md | 42 +++ docs/images/dashboard-flow.svg | 96 +++++++ docs/images/dashboard-leaderboard.png | Bin 0 -> 382029 bytes docs/images/dashboard-overview.png | Bin 0 -> 459488 bytes oellm/main.py | 2 + oellm/push.py | 318 ++++++++++++++++++++++ oellm/results.py | 7 + tests/test_push.py | 367 ++++++++++++++++++++++++++ 8 files changed, 832 insertions(+) create mode 100644 docs/images/dashboard-flow.svg create mode 100644 docs/images/dashboard-leaderboard.png create mode 100644 docs/images/dashboard-overview.png create mode 100644 oellm/push.py create mode 100644 tests/test_push.py diff --git a/README.md b/README.md index c1c1731f..29f73ef0 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Schedule evaluations** on multiple models and tasks: `oellm-eval schedule` - **Collect results** and check for missing evaluations: `oellm-eval collect` +- **Results dashboard**: send results from any cluster to one shared web dashboard: `oellm-eval collect --push` - **Diagnose your environment** (cluster vars, HF cache, venv engines): `oellm-eval doctor` - **Task groups** for pre-defined evaluation suites with automatic dataset pre-downloading - **Multi-cluster support** with auto-detection (Leonardo, LUMI, JURECA, Jupiter, Snellius, UFAL) @@ -16,6 +17,21 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across - **Plugin system** for contributing custom benchmarks without touching core code - **Automatic building and deployment of containers** +## Results Dashboard + +`oellm-eval collect --push` sends results from the login node to the [ELLIOT dashboard](https://github.com/elliot-project/elliot-eval-dashboard), so results from all clusters end up in one place. + +

+ Dashboard text leaderboard: score per model and benchmark, colour-scaled, with the best score in each column outlined +

+

Text leaderboard with full evaluations of small public models

+ +

+ How results reach the dashboard: evaluations run on offline compute nodes, the login node runs oellm-eval collect --push, and the results travel over HTTPS with a personal token to the dashboard, which checks and stores them +

+ +Setup: [Publishing Results to the Dashboard](#publishing-results-to-the-dashboard). + ## Commands at a Glance | Command | What it does | @@ -23,6 +39,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across | `oellm-eval schedule` | Expand models × tasks, pre-download models/datasets on the login node, generate and submit a SLURM array job (or run locally with `--local`) | | `oellm-eval eval --config eval.yaml` | Same as `schedule`, driven by a YAML config file; CLI flags override the file | | `oellm-eval collect ` | Aggregate result JSONs into `eval_results.csv` + `.json` + `.md`; `--check` writes a re-schedulable CSV of missing jobs | +| `oellm-eval push ` | Send collected results to the ELLIOT dashboard over HTTPS; also available as `collect --push` | | `oellm-eval list-tasks` | Show every task group, its engine, task count, and n-shot settings | | `oellm-eval compare ` | Diff two collected results (files or run directories) per model × task × n-shot × metric | | `oellm-eval doctor` | Diagnose the environment: cluster detection, env vars, HF cache, venv engines | @@ -273,6 +290,31 @@ oellm-eval schedule ... --venv-path .venv --local The `HF_HUB_OFFLINE` value is read when you invoke `oellm-eval` and baked into the generated script. +## Publishing Results to the Dashboard + +`push` sends the `eval_results.json` written by `collect` to the +[ELLIOT dashboard](https://github.com/elliot-project/elliot-eval-dashboard) +over HTTPS from the login node. Ask the dashboard maintainers (ELLIOT WP4) for +a personal token, then: + +```bash +mkdir -p ~/.config/oellm && chmod 700 ~/.config/oellm +echo '' > ~/.config/oellm/dash_token && chmod 600 ~/.config/oellm/dash_token +export OELLM_DASH_URL=https:///elliot-dashboard + +oellm-eval collect --push # collect and push in one step +oellm-eval push eval_results.json # push a file collected earlier +``` + +Pushing the same results twice is harmless, and a failed push never fails +`collect`. Alternatives to the default token file: `--token-file`, +`$OELLM_DASH_TOKEN_FILE`, `$OELLM_DASH_TOKEN`. + +

+ Dashboard overview page: average score per model and modality +

+

Overview page

+ ## SLURM Overrides Override cluster defaults (partition, account, time limit, memory, etc.) with `--slurm-template-var` (JSON object). Provide `SLURM_MEM` to request an exact host memory amount, otherwise falls back to a default of `96G`. diff --git a/docs/images/dashboard-flow.svg b/docs/images/dashboard-flow.svg new file mode 100644 index 00000000..cfcfc6da --- /dev/null +++ b/docs/images/dashboard-flow.svg @@ -0,0 +1,96 @@ + + How results reach the ELLIOT dashboard + Evaluations run on offline compute nodes of an HPC cluster and write result files. On the login node, oellm-eval collect --push gathers them into eval_results.json with scores and provenance and sends it over HTTPS with a personal token to the ELLIOT dashboard, which checks the token, validates the results, ignores duplicates and shows them in its views. + + + + + + + + + + + + + + + + HPC cluster + Leonardo · LUMI · JURECA · Jupiter · Snellius · UFAL + + + Compute nodes + + offline + SLURM array job runs the evaluations + + lm-eval + + lighteval + + lmms-eval + + plugins + writes result files to the shared filesystem + + + + + Login node + + $ oellm-eval collect --push + eval_results.json + scores + provenance: engine versions, model + revision, --limit, quantization, submitter + + + + HTTPS + POST /api/ingest + personal token + push only, never pull + + + + ELLIOT dashboard + one place for every cluster's results + + + Ingest + ✓ + personal token checked + ✓ + results validated before storing + ✓ + pushing the same results twice is ignored + + + Views + + Model × modality + + Leaderboards + + Benchmark catalog + + Model details + + Score history + + Runs & provenance + opens in any browser, no cluster account needed + diff --git a/docs/images/dashboard-leaderboard.png b/docs/images/dashboard-leaderboard.png new file mode 100644 index 0000000000000000000000000000000000000000..1a5477c53029c46595f37b0c448043b4816e738b GIT binary patch literal 382029 zcmXuJRa6{p*R>nmrLkZ^8h3YxMuWS%ySr-xjk~)`qXB{icefDS-2(walFj@4`x|vu zbyBs)taYz-%^9nzEQ^6kjQZij2Ml>Rkot!Y$gUqgzz!h8zTat=(WZeifBh=BneGt@SnYUv5S=@}el+N~uG(qcc_?9fCM@hfyN}q%9hh=) zFUIZKOY5zKn!9H`)xP>E1n@!bwifW%`g13_9*#+i>ttWYGRoU4B*_64AKv%qXZuao%7g*zSxU zC|uGC#-q;%QI}$4>y@%U3*cj;)U z$}|oDX$YR!*JTDj$u@yIxp}fIY{?xd20J#R2Ya_RdV2@IwWw0D3jTeSBsvN=@Hdq zpnO1w=Gj-vT??;*3L7or`C{GDJem8PV2fLq@*|j_&@1gkrfBQSH;-OTj=j$Ik3j_x zF{Y`KKJ$7o(~`!WjgB!e}Xt4KpY}aKd&D@_x z-_z7tszAYh!&dUW=E6t6E0!^Vk-a}0n+h=qlT5xPEgd~A7}Z$|ek(=wqs@V^d`U-s zRbQN;wMA8JK0|~izn;#&Fp`;62cC18f3EwS))GQviH}6~-M_6_Tf&&N4uU5#GWNAf z8d?&eh6{O&euygOSNy>-CzRy$sfA1hn;Xaacj#2IYU*X%k*_`i7cI7f-VZCO8k!lV z3$I@Y;esWj?XwK!zUVaS7iOu}KiiJx2S+Z(gO}3g?L=(bP4vKXl^%py{NRaJ(vSD` z2c5Q^b&*V1OD?GXoq&&zR$)xn$qLF8oX+~i9*Q=P6q{dZK`mh_i|tRCeHRo!QVTd> zd%+bYC#@@N3Pv4rZEcfogeMvSs&Ra+iZpK(&tB;TgR{7e$5~44{18r7A8nF{UT6pY z!$VUh+>XRy)A7yK_VpE!f&0|Js&k-jbqwV4NmFHXYAKCGcx`&0qv6Wa^)lo86}NP0 z$=B27-tpfD>)CzcBhvk{OO+CoA)goO_&vFWA2B7P+!KF zBYll!%m1vh7J5r!T`Zg#l6?NogZXA{&Sq^eSMOc|7fx`M{bqo1o-A{M?OF=e7nO}R z74xuSHIA;gSO6aHh8_*;0*pDh8FUzeqfu?a*9nT+W;a!I%{+VFhq8NF(i+bSYalJVkv zzR*MT^zR)}mNJb?$Yfya`N47;TqTlVY;nh}U%wncFUFIya;1H1HS5d|l z<~v3}j-TR_`TV4uH@ftI)L&7hSgG@u_*V%Plc}wW*DY*+?0M{X>l`{hX2Lq5q6?wR z*_dClXkp5Nlny|tIb}KL8adnNip5+D>>{Z8|hpAvl6BNW6s_1+cq_l+dw2*D$;;s85NM|V+sQ6 zP<)4L3hGDB`q21Qa4`seT8ay(PZ5p*N^Uv@j}rsX$p8RJ%7+$lcp>m9YRcq4q9|ej z2t=EK&W{4fadqqe5Ot1u%Ype<>^yT}YTC8MTXqggb8!MaIY!0>?Xj(~vnhHAP6k+$8FZ~4{eImMlVW5)3x zS!t(f@|i-tc)jpk|OJOof`ovFp9BIc6YmuJjv#S@#=m}w3>B`mr5v~~7*O6&FBY-VY2-PMpZ19gh zx3RSjZVWUG)|IWcA)O6Xc#KsJvJydEs^xvE{g6||IQRn5Fc>gm(9#=@QGhTFIkTk1 zz^oW#kyHA-kYVt1j^{5Y$d(UTrv`G?8@s7l5(@Jxluue_pEu=@;d<`#NF$myj@C$O z)NWRPtmu3F`u#wq6UB}O=frNosSo%ic zuZ#?|?6FwRl{L2oa6oC4(q@#U!$lEGygC7j)#kRCG!WulODQQw6A{P$SK1@yF9DM# z7Gkx#D~?`=Et_oThT4MkyAGvrfXo z@(Q$#<|b>OR@kpu9o`$*GtIN+rt}33FqQD3FNp;ctJetcsP7yrzW(@NM=gJw8D1Fr z`J|Vt@hb6*8d)%G%AcTPpOYn9xi&Ju1ej{b^~>G7ClBsUFCAAfbo|(;gQDdh_8q0A z7$D_I+|GuLD4sjsjxpZON`!R4{NxGFU-zkmBX+wu^;O%bsvad}<7bG=FK?H*g=2kb z__-x|#6#oLA>UsL3L_pO;>&YS&Rzkbm9;|_0c5J`2Kj50e@OjIEjrl@f(cyx>FEN5 z=|b!T)K5NbOXZ(3py^?VEl)>9sR}`PNbf4E+7G+tK0H$9`K9U=1p1;XZ={i76||8K zOz7g7D<68)6|`tZi``P=ldC>)O_)4&YoBs?-9iNFT8cuwru97j=DO2q!N~@%XluISPbb%nOxHWFTs1}1Xn&e?x^}*hCk&F|I4s!=M3rx z-Qno3046E54*EOZf3dQl2;d;^@h0NM3JCbM4XNhokW9V?b%_ku?{6G=S;_yyesG z&N7o+r;+J3H-_iCB4}{WC;PpFs<3`;%B|n0b#G z*bb@e^ao?`83#PWThQjs}Pqtx`}~G14ek(@!6N#Nlc zCpOb-S$h!?!wibGx>d`HdytX30@Q_5(%-X_dj*d}pVi=}?<)wo)hG|{ITKf--DbWA z5b<&ys`L;+T`fbU5gRKl_YqRmd8&I-IThMP#>j8wXBhr|pj7z4@%Og2AzADcED+oA zbBcCBWlB)y<=|*`eQsc-{ox785}=sX8fURY84MX4{(Nnp zdk`)7)J?~&Le;rGz*0RUH(KgReXDUn0v7YxN%u;GF&VH3 zu?M3Ssr%%9rQTw*_o9aj`r>j{yd z?ej~vsz=+IRsCt}potTb&_Fy3ez7aQEhLvQ3%tpY{SZ1Uc$7Hunersz)oR{1TmOn| zu2uEvK8;~b7o9)kPep=4eX2O~Bydj7p_mtOv7|^ntl+!kNkCWXSIM_0xFW9HAhcj8wO4_ED zny*?KH&x5#i($ID7yz?{^J2{QIQ~JuEQhDk3tJkcPJ~_SQhUzlC8a8ZV#Q_Fz>;Dj zOogKaf;DSkW!0l)LZU2CpRymJ9!RM3b$Yz}jnyf%2J`q?6!pOln-rds3D?r&E;D@O zG=2?jKd3@j>!{u_eqR%~;x#+Co*!jKm*0x*K_Fe8O`h?pe*U?Gl;eo)IA9y)qTTLO z$E+vLlYC)3(ulk!)1;jj(F-8Bt+bJ&5ngVnZB3KHT$l>n8864s))HrUrMY_a@(MA0wyZ&0GpKW`f9!cYsE2!tSB2c&&7C;p7wY~S1t!9^DD9Gz-i+$9mo?-h; zW=B6dOsr-}aNMx?u!gu!ighNJ;7?@biNxnx``1Ans-4SG8KbB8;rGj$0&>WLC13 zwqie2*3TRKhe9e6)B&iklR1o+Qm3Tu5mCjj5p+~77y7>=n3}%drPG-X%q=n-ePi`M&3drU6tEE9G`HE>Dr@_&s@>e{f$mAo!HS-qy$2?zBn?`w#M zDLeXwJiM#2=yMaWMNtRBnOa>*dCk6MhsWt4hLhk!|&;^4R^u~vSxTTKnD zXZD<}i}-M~rWFS0i=z35-xUURR)4PwLk}jx#nc^1(Ov#m@C)O|heZ9^QHny8L1_~= z^Eu1J5LtyYeHJ_93~whw*>f>RUsUM`v{xdFuz zb@f*x&fm#$xo%jhgS(#h=#)dGj@0CI`C@%?`di{VpW#;db?(8Sjc*S9?g{>;?UOk)H7#9I{gjYRuc(N-gjJ6|`knTczwvtbLf*acbH|?L^W_U*lJ{6qFgmj1v zQuR*bzxPyeey_zx0c}9epgTI=U+> zq6U`+=?8RD?3;Tpy18Ykx}&Vgi*}@n$h|MI25t)wG$2qHSPB57EIMd|ZTCN`guxei zg@oM+mT*ypqXL+*?=++xK&q6bd0X!G;dre znY*Jd(F75J;A8s2=3yXsXxZ1z{FMoepx zaou)UyKa!YGrMgv-c=T0rc1v~JxJ2P&~gA{8KqWc@Ah*b=L%$-?#>X@ zVB*1{RO`5U&axdcDEh?#rBY2a8b51PCGd?kze#Wp{bRGhU4U@{4L3f5ek?wiRqeHQFpSz}b&cF=Rb&RlE)ii1{k={eSCi zn$n$e#|57zySukT3m3b)|8a)5_paldEcqz)SYj2{dZrq zW+V^I`Hg=*B9+(6d2bMIv!jgiNeM5CQ06VbL%C~bKZ>m5a#HedwGe=E zQ;I7NH7nqNztI-&g;V{dRK3y+26Sk+fiD$X-|@!erJQ^ZA<`B}ck2OxBSc z9(ymMfh zY7)8tJTu!v8vfj4>&-uRY9gTwD$KbpWesY<6P}z4zYyNhK&*=9(`-+iLpI?0}ZnS z$-vT)4ZMtCf$i0P0cI4<4$962zBT*fg%OjWCt)fu_Zy<5t~iX13PH@J;A7k#-4 zTlv`#8!&M??OeOy10T$JO$z)j36J9|1!6!%1X&^hKu|p@AQzAl=&M)<2)o_)LS2oN z6Q{fnp2R8}14n5WF8Zj`s_5~G+VJ_sZG5OoWN$FTf7w)$JN{0VYJ7k3KxNOVldMuA$+IRoAehb z?w42h3u1kTC`V4vCl8jc=E5cFzs*r(qOA(O7a{wm1T25uXtYtpYfXjdKt7A zN@z;pWLL2{D)Q&d+bulRoVr|hPR<*(AizUGrF{w3AeT-$vtrFhMo}(UHxGs>#aAC5domfN;CM8kOnu7#5N9}gAqHD>SW zQwHg7Br3Kbgi-3V3Tkpq<9ZRJ^nho-1X?ZDrEg6$oUM+*IFGll#J!K9g0}QrPlH>N zedMZPLM;KWKq8ZBoxt9q=ip#?dsC`Kkf+t&5trs}-|E4dE@p*83=ck1Ys}B;CgB7= z{I6TAz5|jfla}9C7M7FbXM=V^1p3}ELalMVnD0Fo$rsj!Kal7su7sh8bG6jJxa|20 z+)(;7X7*)3_sp~)g0wvUcbYhxtU}RpDl&7_g)Okl@EP=eOmIdeS1m8)`ZMAulr0_U zX%6_q#S;TIl)v@CG$3{G9nt_*B>od<;VQou&8y-kU+VGR`Ix(FB?bmiMK160aO$=6 zxSWR7>HUcTR-IGXTw2nXO!6Sn7Xz$<^uXFN$&r|0ytRtPdB-Yds^55%OvJJHgY6C0 z?z#~tJ!gRi(~d(?stR9K9pP5eY{Y~CLKun+rwe~D*W$wrxXKi3K_W+C`( z@Ke(z_C5#JKQF-*r)sM_rD}gl1&e1pZb}nWKj2}r9m-s zx&U=|SgJ3X;KV}@?`n3*D8OELXgs_ps^Kg9iZIgw>~l`~rBb!4#PE?|yr?H(fved@ zv4+KVfFzx!_4O1CU;_M|I3Nm{lhpy#CIcc$QOnQUNU3sF^+$ z?C3r`a6&e;?Gk7R3v^(X)PRi^rZFqY&N9ux#M8ft1}AcaN-{5@#`AJa#S)l(M?@qb z%7TDbe)i15sNcT3C#xQKf#WD>c-sQZM-%x!7Qj@I2U}F;Q*xRuEaP2x&Npe8Ib!P9@KCtuAH>fY zzh)$fqR!c#Yo>`={I-HXuO+1YGJXm$W&lW^EZHN6DQBQF1nYbG(`3&2q|4a#Zay#at_;_Yb40K|=ovwZU~ zr9@sdy-MHG^si!}`lJSF6!%FDs|mm(Q)5yump=VfeKI!fVdO0;Az%8Uy(y8&gGhUQ z#vBc*K>+t0l!&-)N|5SrY~)E z)AuBdts+QenWo!R`^~apP`NoOxsH@IwI1UyPbKCZj})cISoA@l{;~Ky26@;6t98=L zS>&SIOQ)+0pJ&4>BcWR>A=3ew@2GpfpmxXSdPd^1;4?}D&j5OZGbn`#+i{3*PEf$i*D>X(yQWY1>20_JW7YmgV+D3yZ(q_s ze+M=d4*4F1VsSB{I-^#2p^^hsKB|OPj;Y*t{Y)rh36bUa;g)A~l-W66B8&2IJ^x#w z`gqxT(kNl(uI%?&8HQm!&oAPZOLSP(-90pDk6rmP)Zm=`D_rb7j;80{TR-~mEYa*{ z5>HJ+4JcV<#(t#np1&)rY*TIy=jV^jW_ms7GWN9`Rc<@~{^wV(8z@*eG?VO5FaODv z44v1DBPEKXJ6|U(A~0_jbH3i3R)t zzcH{bh4odaw!YH&b!W}eY1uQ0+jWCbjw}dpTKapvZbf7vt%gi5-RYV*tDIk2nY;AI za~I2q6}Rz>PKktl%miBFCu1dUrv4w%)Nx9KS?VHYuF6JA-B%2O%zAN<+ZhH<%WSP4 z{N_MHVzxO;&#>8oFX>pHv7GWS5qc?M<3j_4Y#gS9l?~%QXr@AkfR8zcAZi@pa0otpiYjghQUp*!k% zQaU>^TJgMlV70&5PJw&LL&bmO`~FtAsr*Z~($1xJYy&T6hcJNpNMY|hZqE|ehjbBn zc&ccC1nnz2i48v&&Q58>wqp`5ebcYQH6-u}m%q0%HWgtK9-iM9L<~;|Qn`Ux4yPxw zXtS0l!pw>P2hk;?i*6a=VrJY)7(M(hO<3`i!_; zJ%c+Mm7xmdfBrTJLrrpS8zKAoAF@#}ghOr(jA_B$l{`;KP#EPJUimFb_=1oXbz&P2 zeF}%06340Rw#6ecpH?MX(N^i{0==%LksFw$-Uk24$1TLVMF2-hXmD*mAYkh?SQa!<5l;Y zGYb{1u8F-jzke)V+cQNK(Y?gQ9fRn1d%T>E@us76R&weh-?H&L8`W<{f%_484Ycy3 zV=zqF$o3XBM`#mReyBK{cqa^wHuUA~-@ZqIQz&oKTjPY^>1Moe#-WBXM;_@N3`bJa zkRD>Jur$iP#%%4q&S}t3-%uoIkM>(8DUAh#EI-uK*1Hu^HOPiWZOzpfUpY|EjzXIX z?vHBk^lB<&5k6e_%$b(LKTSl3f5>iwID-e_)K0D$8>2!Q{}Og+O9u4w zWBbSGPSTG|XelSS6$UhNMX&DiHiVY1K086@^6`rBo&J$9DlmYn{h#*va38Pu+M~)x zd=ot@<;q03j;)f?0}JOfEy_NLlb)Z`ZZW-RC*04BJBzwWR_|Lc3qmy?L=bpxT0x9p zF-vtm^nq>x@Iq;o65B=(je@+5`j^vDL-2NS_NK~NCBpOeS_5xizFOft%vsEG5Upu} ztR;U)*tkFjDdEtfhA}Nc-V*TdAz@~Rjfmr>=p)|Xof2aWc_&KBe_@IkeN5xS5*2KH z*>LI*C{+3lq_sr+38<5z_e6CV{l*(g>N3^kc;;I6f231i@lo5uZ{j|pJu(AR>q|>h ze$_mEk;55pb{V2O$PL-_Z%M!H8<;WBb2THaAb>6SPDhogPxr0_2_rtR|qxj9t+++XFlU55$Oa6_v9w;I-~ z>r2=6f{Vo8GpsK~7k}37+BV1)nK1?%zkj&Ie8xxa7qb87$}H3Ov^=Bk_v7G~j1tzr zp2B;RwbjxRPH2IGl2paT@(wy~h@E6JsE6x#<;5)%CgRnP74YN2jBF3`{n12BV%5C2 z-kv#FicEPaV)iKE4|9U>kvpXcmhiI@TLB+TL1b#&U<-ft&8y~dH)cGlHT}`#9DVmw zQy3>ID)9u45)7ddygubAu?1=@GH?lF89OCcHKMLc9D|5Qj879-(MSA~1KXHOov^MG z3;@auFg>h)1!G!6Mg;~yTT`$ylHNEzI#;BqORkvGsD@uZ2y(y+VyW+zj=iUKvSwxb zl~s&)!Ef$2bzCI`p{G9 z1(NKc0RagVG2$;7f*Jf#wXfg_?2MM3e4p@8xDV^Y5U@o?s$@HFV_Cx8s?(amg*YPV zk^yFg^*Nznz3?Bl5j4zzh5uh-Nczx9Nkwn0-pvxtAchfo#U8qq{G0q)z_)#_2|hc` zP9>Y*lvQLIGaDBT7b|V&g_XRg>6E5c^_*e0Z7P;~ zzDN0t9&%HY9H`o8`NEUj_!A_OB2pEqEAdsPQBv^GF3P(P~WZ!r-D|7smc#tVh>3| zu6R+>=2kfkHRH=g4Y`$yUl<;wg4IwoEXAQnhJW#*APVOWjue;LfNhHm*4#H6%)gKpQ90~8 zi|KqQoRaAE!u7y6a;A0`IyobJn)H^xuu-vUt}rJ9uBh&Ia2?1^tjXC7@m?jb>^&85 zjCqpq^bPxLZzZDt#Tz|>%YHWfD-3==C4=T=FIPId&TtEZGM|)xg}6ZNx&x(-mMqE7 zdunff{7e%%_5m9A$61*p`<=&`w6z>;!`$H-`C#w7Y;3qsxqf9CeVH$N&CJI4`2!QH z_PFbK@O_z+H8yK=MY5zK{K=CX5Beos9U4P=R7U*yVTf1>V#6qrhkQ*WNf^F(Y$fTm z!!57Hi7>h(*TQnD1#Xd#J(m@V|A%|-zRPj11zHICOgKd8sq8;v zPu;dr(Fa|?F!2)dWWkQrFxpnv%X->ZaN`21lJYVTTYrf=WXS9M*=lOXT19dM7?Z60 z>CQ#qHi-6cN71N>c%FjHt4(3YY9PKkWN&y~fIu#FNQ?K*ny#<>a`D3t8ylKG?=a^a z@s$8cvR)?QIyzuSKgV6YVE8Ot$#8IE8>JN@}e1 zSG_MWwtp#XC=Fb15KOM+3}D**okVC8of|0jsYTw8oBPd>YhxolSnL6a!v;HtCGjoi?{rInLE>${21!@5 zBdn7gdgvHNDfoJD)rXf2H;=bY>PfmBzg?xj=4DUBcC>d{iMhErX>ZgIG8o{CIP(_% zLJ>HE1*~`OL*n#Y9BLM+e|OJn!3TVu<=Q-v@JsIuKZ@`Lba zx!<=4KL@M+)R~I4C4F(cHd;;(ey^=0$wpJ{^h1EV&`8AX_I8W1M_@1s={9=A3&8L+ zUsAY#_ik#E^!qyMN1H4Om+_n*c_@Nn<~NO6wJy9M(H`g*M+H7L{dnPk97OvAN|E1o zcys%wfph9LOv5h1=!{yTz%48UBx39=l)KOt;udv^y1*78Dudz)y9N*!1wD!cGdB8; z5)txbWN`_idL6?=N?i3zR|!gchMMBLTcu9Vah|DR1}`ebX9oWSDl{L9s`)mDI@5_9 zK;(Jf5}Be|IKovg@|hOGXc+F#>jqO~N%|#aVFC3MM_JB%j{cam%{iE!)-zDw!acW_ zHUQg<1N;3&#Ni=wdprQZCB;6HRVJbkBDLyEKg3HluBObrEh=V?Y|MNM8RwQZQ zN*LqoKB51b4Kphx(^+8{GZt%@5D2fWPnT_6Ak#!KJ)}qzlf^e0?}#mZpsK=p8Sl}e z(k*;B0&-VlTO^IzW?`i!xndd(##xNZHs*c}M1R?%JxB8hui6wmCF|j_|34Ox&z_7H zpS=A_(L1pC?hpJY&1XA%NmB4U$BTSFsljq6Kx)``e((S2n|lA!zYr0((17ObZ_2o&ws9yW ziPC?z(!Uv4f>*yEd}u&>n@m>eQFDERxtv6|&cc57vXqmED~(+EzWEeRGKIc!YdAbU zKT=Px^*i?7IPG($D&5B}hLf5?9?ly-)V2eORk?eQYj)x?iQN+~qKRMmJm%r5mdPlk zt_Twf0ZDn91t-5-Ni)T46z9j)rh!*jiggR)C&kk1Vyh_9_yZin^YyBjnd}WdE%;pt zZYD1zjAZR2fARZ4E*himf`9r$fX=0-&5FaKQ~>Hg9Nf{AJQ>(T%0F4dPw=+T@pRFp zXUHo8fq={b9hdBmik|xUdz0RN`hkA9z5Q~$qq>mm|5kc#<`v+eezK=!Ah+0{_USBC zrkm$}NJ@jT`{z$)A6eTMlCCtkPutZLNeH<6xBhp$XS$+pgPxA)l5aV8%ip&)1BK0~ zE9{nszG`L3`M%W32Oas%<9^=7LT{C{sYxrlRX*Bp#^ySaC?FrCOh_kC9CjV7m)lR~aHm#e``S#L0WWRCxkI4DG zsTRm$ZTUN(nWr+UjGI_~jr_|`dB{|TvR z?QUGCKbql&ps#-3j7WlTu&XPUO%Z`EyShY&@CNFg$W^+LFN^)!hBjArj&lTk0b zxPyNOxsdXp%B=1Urs|lBfT5>Yh{56gM?==>qdfXGv7mc)QI73kr5RSy`GSO?^!2X+ zO$NWWJ~ovH`APpa9IGCzAMLr);RvE_KaD<5a6o}p;O8ue!@eztG_kphn@|BV}OPe*7kl+2~d#@?lh482M=!T3{f#3#YrL<&hIQfy5!sf1$FznJg~L_EvY3ma^LV-if_l5w-T>rJrg zI4PwP)4n6EjzL#QVig(5=YMMbiqJj3RYej~ffLN4QfQOXBGsbev3uryF7GR$y)NUElZxB9vIL5fH6z6S3~xCSG0 zC)S0h-HG7cw?t`T_bA(5P1mRjRiBZ$5wNwg#8c2qZ9PdhtJ11fGv272oGE{N?O#WH zK&-@W!eOxb=j@fs6j_rBKjA_Q{IWqU_YO`K1TlUv5jC>M6sbQ7IadCH^=%h^HoHnW zQLxiBy{Ut$z&UXsv4(t)SfK=a)-&XfW*5g7xS>xqkDIm^-R0_dj1K7#^c2n4)zcDD zhv*`V#-ZRv$tarIr^Gm`qmqWvA;+HJYh;nWFRRo=4B0Uz61uptVarrQxbA7@G$93f z*_mCjzY2D{r9kb#4}* z=x9W1G=ap`w84|9RNh*8?Yh4eiN)JFS)MS`=OOsvYBG$M(x#b7jwsd%>+Z++$zKto zU8*~)ej1+1Y!2lb^$W8N^^I{IU_>t7tj&oy{>Y1KrieyTEz<1ndnuxFh>-k#5U$?7 z(AC@?{v2Dq1>A$dRPoM1=XyHHbracK{E0HxxBT=q@nYN9Kj0$5CFp+1o+sJ-xO)h! z^CBYY^bjv|jrB<9f4!Dqpj|Ms@iGnWHR9*}+hT%scmP+2qx|LQ|4q9!b^2Xq#WC38 z*)^|)HFr`Lz^}i0k@ky_){WphMsY*c_4TE-3ME`|+vVQ~>`Bsg$cH1dEI?%bOfrFM zls6A~v=Dm(!|H)d<{@R$GNVUFh}rVM!2yEc0(4G>^PQ*imJF>tNqAyXsBd0-iQ)ZKrWs5o1sZvR zDLE1E{GZyhmeto6h`WC(NpAHrkRlh`6)w6mg{QtqnL}VwG&_+q#d_|hv0;gUZqr$18qY&GPLhetVpW^)%~1ZkOSFp z4yNZ7{<6|MDn7RM_S^kpPiokAFmdY|_9K{U=;i6AnZ@{;Ifv!aDWd#%gcW1^Eqiu~ zCsqpF9A0jBC~_?e-|%>_k(22ur| znse#6D>vV#CZgJ4E77E=tIgrJ&(j*ei8o^7>waUkn@0-H(cee$ereaS6LKU0m7{Wp zy(9PLC>F+s2-Z(xO{i~Y^<>Q#2Ul|a$=j8-d|#*PavdOkw$qP;M{BdszZ71#)>Oz| z>9C&GG{`LPJpK?*Mr>`8QQr;j7&6F|9;`Z`Yen@xN|1}pO;tj#g0*KmU6-^OC*G?Z zFq69ieX8WU|C`vWmGiXvkt2(m%=;W(T+{^(qO+l+kkj%&*ke-YOG5YN2re4s5zRM{ zIDwM(!pY7do#kPO6h`MUCxCwB3#nT%?k=h9SDlTaFnjjdRBmbsYlzwAFi62pqv-k2?{?c3VLCIMa!~9jHJHIfW-G zGRI|}Pk~?x7>puZOW+E&$c(wgp87g26QyXUFk{D_VgvOlC99I%>z}godq>Rylr8Mv z%fOyjN+rudmV?s&O9G6+;u0K|fTIcA7Qpcz3fW{VFQ~*45yC|dWocxu@Co{8!<3I} z0P(PZ5G)K`)lF#-5cr>Bed@i)i)x8|wR`UiJ;Fo;&;3-L<^(v#Y*a-NBp;7Ts>h~% z6_yXW%u}(F{*k6|s?gbZIJu~?YSH${oOk%r(cDqm zLsC}<98*;b{LLec^bg>`^VeB^_!RAiaE_3&0+-DcSy&?IgdYXL-8bAD4vUGaJ~Tp_vUk}~dEb6qW7Uy25|T|}X0rf_iE7*c{7tcK+`Qlfi2n)*T0jF9Bb z-|DSH3!lpe>>@X%BhAAPwGac9eppEE_L3P)B2BzsE6uZIl^EP!ABKE1ghlZp#KGu_ zX|c%TgcjMVGl8>8!z4%VHhqhFPM+nY(qP@^ISpy&!6ppFF1tw$4YGG1LV&M1?Jk$8 zVZ@)?w^11398JK0Lq%fDN zrn)wcPCjFTPHUj|^zhs6sP36=MYIYi-0@0-U8qQ<#s3xQ_4FoVNBZDK5Y_qnqaT-v zB6{n$he$=M{Tmxp>MlMY|5~%d*~ZfdiH-b?O^z9X{EP7Y2zl?XoywlRjtouSO0wD6&ML}VUdGsaOyv+PKP4eEV!F=v?%lA_Z#RZYf2jvO`d$N(h^ajeW`q@-+1-}W z{c8X7?f4XCfIxcOfAFB9BcEd;tcaowlj(o4Q!l}Du#>wHF?gi2d zmp5eo&AGnYQl$RrRY4JlQoX`SZ>B7iNNO2OC_5SiEyRBfDTi-r<`tdYTMm_=H78`5 z{YzJ^6DFIf7ilbdQ`t?NhwcI`+Lr%(^eaSXY&R0LZG@$iTRc`>MD-d?_YPurp$x$_%%{voV?Y1eVA ze7i|bZ&CMG{3;&SM)(Nx?IGi<68!%VFxo~gG(GB8RNCC7o~_{qCvrJv$wWw0>;~+X z2p#5ppQp9~441vmWujmDti^W&=$^Dm54Jj&8VwZsff9O@;*WM5Vxv^bbUy*~$fX$R z096@~AJ#goQly}-3@&~y8hkHuDr!5CgcUr(%W#ln!u_kb6S{LTEr5r5vU-HeAzDXN z|7GY>_5JEE;&PdpJl&rHuJAA~RV{gMzLL34k9XPbg=qUSPngV2!4ww4j zXZRiQ;cZ09Ys5M3!|LGzhRWbUyo2d8urO(sV1a}0l(U-X9mNaYKYYV{@2D9%SA*dv zL+fRW0BAV^N@;ZFPgN6Hoqof0zM5f)15&N`&>a2fa*!JKl|ot|;ZgBqH6@0P(I z*+10t#7jievpf8o8PzT6CNzA>R6g>lNIk)|+cK({@~H5MYbbudlt-gx*?vBGhj;~^ zcvH-pgc{*%6=U^qt2OC4E3sCwF>$0@y;#E0NHcn(n>LC~_|K4Ucr-1%Bfs4`Y1Mdz zwbK9gc`j%FOLCz-$g~?IE<7`kQjGM(Fc;anZDA#LALMu?dESax9r*j`Q3UHPB6ek+ z4w(O5AG@zCc2%vfEwtx$&h?rnZx1+a`+NSR57Dl+hqkiljxZl1p z2O5t}#9Z=~u{Y{-?Ev1wQJ82CPp-Ii-X}d-aau3G{?O5AGiXr>NgDTq)f>G2h4Y9g z2W6AOG-OGL_0jJ#4f^{z&rq*&_I--7nG1hJEXhx&?)5`<^JJ%dm#lgdRHk4-?wThC zuSV&;661fr^ka5A4N~eT6q((6*Xc3m?~&bQbm;jt-gSKepoo11{zP^6PIQh#1j(vV zM?u2fAW$6_=tow~?$Pn8v-j3i@qp%}tni!y(vwk-$GH_i=bY-hEX~b@nS> zbtn1bF>Km3`(5YV3E_s&l_4-sQFkRCCGX%rbZbuw5>dSWRobhW*RPvqjxs$Jt4fUL ztv48^Z-&BH#{L!<(0z>U^A|rW@;L_Ke8Y(74|1HxX|zrXQS*=N-RHIm*sr(6!wi6zDp^2+C=h9fO>JYE(l>Cf^T7!nMD#Y zo{d$GNgr5pqLy^D>O{`--{z?pu5N!2teGOesX%i8Jnt9_U8{J8+<@RWYC@J|6@&87a`M1 z2HckpvOIBaPqw2eG}w18JT~OokW-N2YTafXE-nM4lqxD$XgrwX*op0JMh74Ma+6E>7KPF7yv*i4RpRh`Aq6Rs3H z#AS!r3=CVt2v8{L7-1~MSqsV{2oUBt173oC?hEbEpeYYpuX>?a9!B9=f+xKBAdxDD z%h7UYI7(J&esd}TNTzdIc0gGZCaz^_`bZ;W$B&`XPcr``>bo*K$_5iv5eE&f89ljr zL8$1b>Cm;0_bMN>UQm|peGOuw=%~mc{#iuHB(y6!3vwKOANhmYEvgSm4Z2&s{;=DDs%(ZTI2 zA9tP%f^0(74PW2zX{dJRQDR?vX(T;WrJMl=thM!b zs@--LvwXhnAIV?SzxT~N4lnAQ8we~UEekDHm*spP&+bTeWh=#@x4+Q5_}Ena$*9xR zAtr7dZZZXuBe9~+Bx1*vj>kQj^xe9I(1S5v5y|QjQzke)_1~N}s%_uxUTT{LM}Rtr z!$2Kei}>{uQ7)H5vFm6SMrmCw)W)0Xj`49B6Cl|SjHf;E>9+BIX)cwqtguw!H$<9l zJ=r3Oz&#nx0*}8@7@D>F^sc#^`(E~ij}n}>MRSOYVe}|4!T@&Zaj}iqBcN2oQJN6S z6hYw-(N`JXO^QU%tO6mkVTAFmuN~O=y6LW znoz=P38v?1cumXh-}yeWn3FwDZf4Qbq@KZ`a-3G~o>e{!<;PNB#A?lGgo|_tU!Cr* z!o+cxjT|XGvCREvLOR?h{p6LX!@;qau7si2M|?r;v()EC1o_%@>}h^k!*1Z+lkw-4 z9r8E+rHSj+v`ec`d#t$6E|&YgnMrQpvHXEu2pyvu#}CcmS&m^NEB5Wm&nw_a?z@Iu z*QYtU9jUn3enZWQpXRXesKllvWZq{aVp6-6=u|gA2(79V*|d3U=Oc3X{48e!QF%Jiu%X_wFLK`; zgK}ELa_&0#!(gWCONPbZX#f6X;_*_NaAB%q{`Nf~$&%0QRIT+-MO#;i+M&Hk3W5BG z<`54W0c z+PVD>7v~OfWV8De^BaEQUO7wR{$t{0AH$hR%dE!{V!xX>YdrtxUBxPX8yI;kpFY)* zC*bHdnCVV*-F{hA7|d6&-P8~Rx4+h=jU_^|Tl|R|YUcg#C8MDtB=6TTOQ(6lC8v(u z;oP2LuX!;jp)4MzQIq*qdSiOK3b}&E$C(?q#~eUNDuY;3sivEXr0*|aP6aPmnPfRz z`LJ)qtE`9Djm`Ta&YnhKcG{NfK!6pN8o5U6Um~!Z)wg&FVB3o6YoH(Fa_??#<~Kjwn~@! z6&Qj3wivAJ8hJtuj07Jm%gw{W(500|x^Y~xSVzlSq4?V$)x0c){RBSC9Q7>BL{2A5 zhEL=!H?3WkE@LOzio;^_L!;LUMokKtHX`&pK6x7{*54E8*VmwbM2ZblmJB;aVP4jl zDXQrm5ac1pSQLql)NYe4DCTgrdEEaVD{b(|i6ekd!Q50gOzUsytrS$C3pW<`3GHo; z(<y0dGDVssi%SPELTMp`9LlkAq!hHaBBAE^|H zdY|uGjQFS@^;d1&1;lgz($TQJK83l7*zf9*ivHE8=B1{Z{k!MGy^hyxGD9c0OoCod zp$`zl>|U{|yCPL_LWgjVmZa9pZ#&f1x*jhHbOz%v4C_|3nxiCu8`Dn)io%)1YWtwc zZ2?PVHsx~LWI4V*_4hwq99>EyTrF~^_Q6(!{Q^Hye9ASUT^y6D!pW5@Ysgzx$$>1p zGGeTI#U-4KI?=6LpyW@mVEy7wcDnECX^(zUvsj(ClT0qwnd#9Tj_HO%^=M8pxyRz zOAyZ$Q#LRLDX*O33%QD+HAlr@4W}8@notZ9cHPh&R4qmFFY-H*!9hKbpVNZ&t2}7g zpGJjXs5#BUMX|6(;04e|!yfBl`yNgBlKc5 ze2z?A4ca}I0ztywUA!d-ca+-6x}b{e7H~XUWx|o^n1~cu7=d8+LTxNoLK@^~pxty) z>E3z%chQ+-(p@U<`+jn@UN9L-09P!gQoh*Wm?Plrg9vW+&fwsts;$z5oJJ$uUAaE9 zS9cq=m$paE?WoQhQ;6l4s`6H;3G<`ZB@=tnC{?X*;+SfJ@ykVm(w5xry@pI3LKSUy z-(`Dr^KWzPnw2LxeRWekczDPYN*c;3JGD>of8`B@CevyUVxU>b@w{lC*8EcWZQm!+ z5P*nCkvm+HuXuweA7`;N4j*SQg(4l1U~;~|0-sy3g-O!XVP9#+GlZXPcuZyWMT6{Hs4N#g%c{!!uHXwYSIqwo zOdmHi(mTytezg7LFQ zO+nODN-gk|JxIbbjt8U!|Fh$@Hw^a? zqMqy1&vd(Efb3?#lB0y@++nb*JZgLVp<|hvD}CoIHBEn?tBKlo_>H#Xeiy5Vy$I>> zq_??ZX&~53o2^ZwlQK~|tZpXLs9;~tWY0`?bv>8OPihbhZS_On*i!P_7f`~(7y&iE zkqoX{b`#%nXIDZ(E$OOb(gm1wc+T0M0=4%Eo_&cz9MvA`coZS%N&<{wJ+U=$Qt2u>)G)Z+mgaCv9zZg7Q!jjpF3bP1 z074`2yw2OvDBQwgbjoA|fbIiPzbv^Qp6#G)f*F<3yrp}SsI|DAvG&h%ujG+g+u7pFc8?zc}ovXyt@H`9schGqv$+`Y(q2Q=Q0Hn8lvHoTIS$ ze6=?N!`E93-*%W##h+d;l2B7n6$P*P+?PfS605NcD_LT|mz+*mN?iM18b`rt)NSu7 z1qTL6c6@adF@;l|MR^yjag3@gFg0JHwGiRbp<&!`P7jNk@Vtk6g^W>+YFSE0OD0k% z{b-Osuc-X>d&}{;%WWIi_fWIEtP@BrEHCJzmgu6VIa{Q=9Sl{Y$E^`&TRV*nB!Gof`G@CP{+W!q_IEA|Z1%GA#6H~loM$CJ+BDFPbHBG_T^QEN!zjqyw3x#{?y668c& zPimgGsW02NbVZ8_Otsy3Q(bKv1&g3sSKGx?7Jl=>$^Z-h&Zp;stllG;+CGv_*3Tbf zI54CcjyGvJ13u#eI=T=eUw?8BFxs6HGDK!$49ok-uYVgGyUI{V&6Q)gj`l`wo;FP$ zORmttE7-dXYr*5NG#*+1dt#~)EfZ;($N<_6Ci!@>R)JCks&1g%rJqjIWyy15ZW66$ z&*8B0@`N)r>fXJi0-_-O!a3n@i1VHZG{TYApCIN(m)}*ntrRTQH6=|692P>Uc>K=o zxnqHfJOG2jNa3SoN*-ypg;^hE`j^jzZ%!-EV!qE-WjVfrwyA;Sk-d!C`(Mn=WyAM9 zg+oNlhS)6SwX0kENh;=K-7_s>_^t_MD*TmgiY9-ySFX$P7L4>?G}mW^2SPW-ZiZ18 zFmNoyUo8K_%)*QF$sq2$5hZ0vu>sfJTHm!-M*#Oztp2rGfWiVhO$}0ejq+m58F->v zCnwjv_B(Cmov867RnW4I=FTrKmziwwu&hVo?C8Ur=Qp1HO2=%UvhTj~{Y2XsLJK&= zzAas7wVW>6k@g~gUcUM;jjt>p;NBp7JSr5K`{w8cNtd1DYWY>-Zj{~nK`v0Ii(+%{ zifm1tPiEkF!6Qbo5D9yi_;pFUkz0y1wDhG$S7O z0R!be?N&2nf<@xQ68H=NUJu6?frEFPZ*_S1RmIgyT;GID6JkB_w=^t)Cy+z7E7uNS zWNVf-iE`^G8!rgr#4URR&T}!@OjZv1g*8v~`Y_uRFugw$o>k^K(MtX#jN}=gpv?Js zOe(V>(e3gV*Hr)>Lc3x}vim$TNfYcAve4m`oe(%Jv6{aZDNp^|vI*c{g z9|$cU2-0by9zHtECp--B4S4e-Gkmj-u@xb#lQQ~2silVewJi2J`d?KGx7N^<{i+t_ z3TTBz-r^TOM?-oO0vm95DcD^ue%+8{Ovg_rq^9DqE-si@A`hK$vDN{#2}_sO5oQ`4TNNLdo`R80G6BfRHn%|F{m#-L{zb=Cyk|us7jFY{#4R{ZaJ-L6vW-xgF8iuJw z!)5DsBHDE4@dNd^n=&Is)~Hp9u06cXIJxqFOZdx$b5NKcCN*A+Lm zLc9YohG%gPLfSEcKTg%dJba@>q|3j0-9jb6uQQw>i0|PuvK%s9m)LoJz?x|z!v)xt;p+G&PkK<*%iQ=o4qs*&iJD@2`D@UW3im(d)NrC!> z#zrZfL$f9Wkv7(c0b&(a()U5=^ch7jauDyrL#=7l>JZ*!`I7(mKlZF1s~7sfmfgZlhr|Ae#AtjCoV_xSMR9rwo@N{+J&5mA&-a+n@gZ zcqz~&>zVP&OWf%ysx25IW5YZHaL?mGxFRIMeC04+=Y6y2mVRxjVO@YzFO8LO$iTx4 z3|ib9Z2^=UTmau!>FSL2+uvi}=eX>$F3SVu!VfT)AxAF>+lBv&L)H+J(iMNpALj}> zax38dnynGqV|0u5m0HJF8v>vX!$F`Qh0OVYgf?P6hBF21f_{%)oR6mYRCY(MhtWhE ziK?%rAB{<%jXG7W=z(yQah#oWPzN>QK#dSF-`%CRxG^Dv@V3W|H{2gN;9gw_)lJ$> z=;bRgc+o88#rw)I(n(MEkxym_sOv)@tzm(O2al7>3u3#+RI=4k2mtp^FVBtN6iEDL z{(4p@Gmn@#kt&lfI>go9lq5k?a%g7A7Qvk!rjWPr(ZI8JLA>1qi$6>%SOImpoZ_U# z@gi~@>c3P*z)$a-WURiCF6RvYQnOBQA%NbjpO*2T={kRZ>~_NMqy;WG)U{5IuVQ`j zwaLPEvH1X(gHgtP7cyA898H^5SyPdolxDDWzmV=h#QYMwKN|>-MpVy%dYPmuH(!ng zf1V~u7_R^Sd9-X@acH^EU#K6U%(p8e7pSRkHB}EqPSFi?LqtV|xLNkSZKB6s)&F$J(hA>cfG;sc-caACAA%l49#P+oNx{>gMWaC8 zFye{GQ+?*Hh)LXT2cSU^n^9l$;@sk@(6{UW;{&>tO^Sr`?%>&#OZOhkCEBKgKlz^k zTJ9GSi#{=(p+T7~_F)9d-(9TASKtSBIaVYs@c<#`YWk)C`$PE{5h^C_Z6>1GL_qX1gl~h+_c#_ z$fxRp&>FyR|&w< z;&^Sk?_XDY>bRBvqa1*m!~p0ROF@$oOmzlN=SJ6JOL z<`3WUeAmgdb+uwPjAGMqcDP>A*d{CP8Ub09oPTWquOj2 z9jdCY-yMrvNZ0FAGmn%a_e4}!c=Wfb0EexfEiv%p3@tg$8MTT{xTbkOvYj3Zl$<>R zdynvM=InTOE%$zh;V$Hy`BmK#<+Vm!f9+v}9f;vDch)AS}>-kjj; zTjTI!d6(X?zj>YZjZ4=x**kaX`qF!!(RBmhg$wkk^Bybm5ZnDapnE2?@dyQxufqvg zSXTV7{`iA9Wb028S)`3##}nVDP~S>L-gd$HppEV-*YZaZmsJ!>V-RSv2 zXC)oyN1cxp?5xQ7^9`f^ZyD}DKk-@X)LUlzi!-1IqZjTGBoBsQ{(r4a{26o{RdUtY@QydSDh>#|zVyH-x*$zc$C$lb41#DgwEbC8vMCyYpBt^=In zt;(N;9+|(y?)+sp=t|X@kXK3toZtCoGN)4%>(F%HUzbdq|2HO16h}IBHGg80 zOQ+ofV5|V0kNg@5FO;2%za@UrbYHyZR5BgQ&V!E8VE_|1-WR2mAJ^?Wonc*k9Z-&{ zHA^rY8s9AWp1E6V3Zbq4m(#Z2?~3QJ%p^4@n<5|fK@p9Zlz20!=Ffpqg1yRT_ zZ}ysLY=>!D9??Ibl#4)hdS7BLJdXBdHSIojWp5A`cLs2a`^6LNolSA zHO_%OLz}xMyFw@khu-9{2>w03alhcTL9V$jq^1RUe7fo%IXZ|b&&0~daFU}hILhGq zUhxvy0d+y~0@v(~C}>A?tzX#Mv{$j`ZOAd0EKK1|6yP|?nBl*9D=E@d7sg2pC)kb; zXV}PxABduvaGL6^b+U5`+0()_gqrj!Gn^rDv3efqBy5F$#~s(Vo?IV9#5Te&R<5KV z2iC#swX()|qf74sk$f*cN&ntW0mwjssT8l_^p`2hQksp zVn&Mps6p{3Bj`mT6YCNJ3MJZL+$ks&|Ja8tV7lo3j|IRYe?c(lsqK}*ZF5)no=vF1 z}1})KRvFlS#5#+BGC2j)v zf&5i0i>DP+b#NV@t{DiEaoIOagS%L&8sqLy#`X#KpTlfcKK?KhU(-pipOK70(#9lk zPNuD)>8Fs$*sLv5_&h?Y{Sok42qFGQ4^WEaS(M%dCa~&lO+3QBQXmwRedeDsi8bB_ zAgf82w!m9(rQoT(nWHkUA5~HuTkG${zg#P!XIFLV%_+3lD`rrk*6B+t@9|;XE9=P) zkNcszR|ALM-mD0eA~&?IlUzSR;M<_Q60FM6isednID0}uWPAupo0?N4$FnR6@h7W| zvYn(lqtnjlt1gTlbgroAV#d!+$XVTxFnC`CUygVaehVNEzBdyee2E_WbAC}KXb=(b zW;ROb416GbU*3F=eIEghz^-Rz!zyQ{Q>te`6HTVzt37lYr=4l@p3iWMtO&p zljqM>KN*@DVZx94>hiYeb$1;;#pO03zaxBL2p{vhiLsXO(TtTvFs=uNi15C#DVcCF zMs8o>6)XS1xo~D?!~}f0i}OC7_EAZD&izkTm`OeUdtcUFbrrRNhcCCXP^Do`1rzp6186ZMEppPym z6+HVxpd`x5Z6n!W+NX}lR4Zmqh8^*VlS6MvGp7qudgw`V7s(r%lLlN@IDFX09EGhy zFS(xDK!c2#gY8A>uZHp`@b(m|mKmDsN+DK^4R)!;^eeNd>dgoi(GW?hOQshIGpp*hLuA6^+Tq&Sm~Eb z;#Biz&n6uk{rQcml~t_kTda=>6KhW8hgtzR-h zUlOkpECaKzeMlc)onOnGyMRh!uiSE|Q5Z(^W=N?(!(?Q48 zK|`eeH_*<8GH+9`u~*8&8=sXl2sldTPH7q`Awgl#kh&!Xhr9w`m1kIq!|*2`0iVs7&fdX?J&srR^0eH!7`=5}#Pkx}XehhO0Pox-!mdTTE7a z`$*9g?-#668`7j9m#_-Y_RhB-(CLzay^w})&qjD%(~rO6p>PxH7-iuz3&D987|cs- z^V_c2I443?$Qj<`)pTeLavT4&+MLc+bTRwwW~H=*oF`bm*fLQxJ_XaU^PcGSlo z=_T)gI4aRqcSS$;%!R?(Zjt-XQd~2;c~e=&@a-5^SdJQ6d&!f9eago;)6G&Z?RG^W zV{||5I{a^q$`BlL9SqigU`Cf_H43vth+o9`{^i(3nB%YlgsqFS*2T^LE?_GTAEiYR z)RZ^aT;XFi(n7ZXoyI0ZqeOtzWNl##Yq{a|C&{OpGh@kpL~`Co3$()(I({b-LHv~x zqRhz#!c@}z8m}?E?ki{y#UkXF2i9$KWe&g`ib2BzVODk_ZCk?R3FvsmTnKhFun z9(TF&)LD|5bFI#*G#s;bLoc?iHYoZsG&=n2%(t7H)V?rWqWr}8&flpg1Q#~k4?ROy=`%BK!^|zsHjG&vLI+`PXSFufwUSmXW`ZFx)a^ zqSMN~VNE3SJSGU--<6}UVsZw!eUzGv1kgxj^er~#8MtbRl+5K)<nG)+1Cd= z?=8-6KE_wbO6e?j7iA7N;G>uBQ$Nry>hy!=Pc=hO;`PqJek}dsMV+W$==3^E^`N3MFW!8Y0$J-z3(mDW=dkZ3m zu`6eHKUb*M#0(hkZoYzTx1y)xP@Elg2W&5_w*eP&)B*=M6rBya_>+t zBBAoB9$qyO$w+D1QBb46fFJ{;RI>pH5X>76do~0V!Y$w?D zYqkVgN80A+slC>Zd_m&Qgl{5!|GU_?{v1=tAg!UETh)Xg>vhc&FZQqNS~c;e;o;M} zE?b;)GbVfzXx(ep_1N7)?ayge%ha;ZaR+-D&F|17c(?le?EqA?S+jec*dx#Q+As#v zmY2{qx$LCget!ePMCGltJq#=nT_W-|9R89ztm@eSgyk}4QaD{^gq9RILOW zz1aW!>=CQ0)S>XcL6VvC$^@I83iA;0@QdXU(Ytcks&EjMV< zSzt=k;}xuz;fgWnHM=3LH3V{^qrS9p9U;iX_l?y2a7S1hF!$diA_IU1j8)o4Z7i7l zXxSgwwFrrBlAwqiW12Ci_#$7GJM5ziGZW0YT>7SNML^@zG{Xs^zlFavns7g8A>r#(0J_t#Ekxjj|in~h5Uv3 zaN1bpo#6~NSl|1gDyh!qv3xDDy*MOl6GZ8&l^K%93FEKZ?>|Fg3OvN<|2i13Rn<}! z>}i;_c?Z=!R$efE__t(i7k#CA*)IAYX!Ssria_(BW3T^>!cYky!5=4JI^fJsmZiV8 z*>Yo%PbnADPC$iA9G_=l}gYB7~I?CUATKu zX;9jduMo<>u_&-Ys(lQ`ob*}gTZl>(1zA4|PH^*i_}Cd=e#!?2qXu5e1ntR)Tt%y# zZPXk_dO7DKo?j!Ux3-@UG&N%zbCtF>a{#W+uUfpxNin}|i0z|6R4p3X_NQ;3D518(ej+@gU#eRnua{3wvf3R{PbpyDj_^6K7?_`}EZr%wdL>TL6#ACwUEN17ACa%(ktgahO-g+@e-sc1_kmkMo^$$sr9(cK| zN$|PJ_mt{)d-p=sdAAYPPk+HBSwbeE@^iRl&yWC8^Euz?O!GbA_D%-Ql15wF&+PI} zU_fQ0MBD8NNMH<|Y%x@$>P}vgX7iZ#;u*8mYVzN9oRL2>gzew#&rWz0te%u?mQ*JH z>2o;b-+Tz-TZzjZMX!!^;xV(WKrmBwc;Lvt5%;T+rPQNM4OAUmoc+IgZLemt#k$Y5 zDd{!b*6p292Jwq&K*C~;l`;f@6%B-!#T}g+ldt0T*()Vk`@Oe%f?glT#c+KM_kX>i ziD)GXh7rZ$u)!jZWyzcHTzYyXp#DFl4k$3osR2re@X^WSlF{bL)*RMPt8LT4{T%v& zwT_M{e;ig+HYV{Wr9D1Nsq-*?0=O6&Eh4i-^_&b2d$654GKI6^d;5V>(B+IaA32UD z?J4puqqb`S4_FAP8(r~zB1!j^NXm7^XM!dydMk`sL-~=;kJdqvMw@3xckL5Ar4on; z6?RX@cb)WRk@6i)3FBLZ8)HcauUe#}Z%W{J&>QIWttXay3QuTQOv4A(w#RGU9^b&f0FLL<;o_BsM-Mk!reG&oVzI%ZeK;cp^ zQpR^$K}V=>lLX+^pcfwSp2{mssJs#w6+9$zci_j8^wuD94|>lAzk~AkUQ+;1E6&el zoA;Bi9mWqnB8N9C>7?(4q|Y2d2a~V)?~~xn{P)WI#?`odgZa2y>f$~Itbs@8t!eU1SnZxR?AN# zQQg~_H^y9E3KlPnbz3D&mEPxR1s+0H0!RvM*8j(NA3Rz7k8a05*5JAq7L1b6R3j!^ zA23e(;yClug9k}e2E%mdgQj(i4V5?g*&^@VCiE5RW0r872DKaVvwML?MiIVTd44_e z`4&&VUPBj}9X-nPI`jy20^4DERdo{%ZKKkv&0E5SpcML4~m`qW7I8Q+3@ z13X*|&8g2JiyuVJtb@U1=+4${^pe=LKmkz+&ik9h%PyR#T zZqV;ca0meMmY>CY(B|8^ZtFwP2ILoIG0<>#^X9UL)HdwUzWX2(coA^nXA$^U-(WrwvvT zOMZE$?@m~X+QEhjWNQOxsBO+0(o80}^7|W|#bwd=8;El_-sHQb@BrRmc$Uh`uVmq>!JmNa>K)OV&ghVK zKJ_X|>yqBRet!`YPMc_y@9@O3!6-i3gVa+(?sOSe1{CQ^?)*5Rks30QX5Qhkc0ClF zrjX7QlOsiJpPtgg8X*JK+2S@N*@iL=F_B@sGPbk)8Kp7OiiO?iJ7o}dJCrh`)aBR)-AF;aDN3g5Xk#km~2=?_=eVeimuY&l5Y;ClIPa@ zCx1DT*dwb%+bTu}7xy#kbCrV;RhC2OBO3mWXx%l(r&-;AWDCFo&b4I<(GIk}!bV@X zMl7lW6JJ}0k{LPawov34i@CUNj$xq94IQT+9yFBF`zR*-Ghh_Y>= zK{zdzdETmX@kt7#`n)n$LV7gnT`Vs0#w8CvyWV_#jtDvc6(7Dk#muCCJDh!Y8b!=p z!!1pGl?j^V5i$VBS}Z<@wY^j7I3~86^p~RZ_V3Kwh`e0ozcrA88!m6mMegrcoev+c ziIh(5->zBSsX>s%VSk)|wbE~b-$X7m-)D@UEA#(u%0FEL92j4>^XqP{s?)SjVExh*t%YiOT8K$W4rz z$Z(eShtqvh!wUhPzk8TkLd=-OU+?e;7OdC-AK3Q!`VHn^@PUicCGNUFErO=6D~Iyp zIWeF3jRz2N~1Y|$#GRwl7!3{Uh(#zgc) zI(<95mzLk`*o9RIHuv)UWcF|rMOo~y#mne9Q)%@SmN4lj@V(0T0>%UI+ea00Lnzpu z@}F&%t|qR5P|@CnJ=L?Hj4&2FJ{f)2_ST~k3&%x(JZmY*DM`DGB~9WC3vT%InzHE9 zaVSW{%zqgm77ht}zO+vGQQ&6-?kRlceGNm7+Fba!dfVch=pLBOaExbmG1_yAPwz z(~LYc+FX`!f4^Sb8u|O530b|XlA2n6YG)wa3UrF-fB*#@SY=3 zQ*7%i`X&FuV}mPtE83xqS*juzP=xh~f3dn!{TgWeH<$d}R=Ly4Bf?`q<&+GtmPY@B z_UTfws*Z(geLA5&%2@7Y7}6hn>SOpJ1!BH=t8w!85+{I?;AKWhnlxw1sT@ z2&F`-Kb=bRaL+5d3fsA-dX3^X%Ji3{;v|Ya^zxtHb92PXjEAvL_PM`$P6dADpfl2C z4#Y($y~E|^rz7V{ht#8_%;%v3bgSP;2e6uKa>Z9NdMH8bn&Voqsi#}J6RDI%BpD3spLlH%WW*adD+?7k{Y1U^QJ1kO)+Y(0w)Ng}bcq4`}GseXKI z2cPD@WH>)9#P%q_W~915mh9-QisuAgLW(k8T7zC9H=d_mua~nBrbOSrqlt7xPgnpM&Y_%n-}yQte8{?r`&pYLQk*c_fSwC7y%gD818G9Y+?=vVZ_!7#%?`x5hh z-!2vDhszT_g1oe$(a0-y6`1r>c$Q65KvGD=ym-5EQdF8gB6+x`kcup_d#@1^4GWk9kI2`Se2~WsQn^07E;S95Zf{_AF z$krO&#W%Pt+?2a{_Ftqe+Ns6-01QjNxKp{_Iph1bm1AMOvTDAV)b0&;thz- zMEK))k%Z89dE<}bN_kYW&N4}| zs67I`TULDGvdRGE6>pEuB+b)mF@trO?Z8(Hx7cl68+L$3*fdW`l6Tl!Dtc@Lb(CqC z!FF0e`+PXK2a9ib_%~jiNXYg+7;3cr?IHGgMF7{w2A~_bA#$I1E(0rD=nu382dRZ1)M?CjK`Nz+(b*S@=kn##A9v{g@?x;)+Ea z{UEorFJxa?6EcJeS?l>shn9(OszGOM!9~G{+1Of0kNNYX7&S~ocyW){JO^v`617UE z#L{t`VX#iW(;%WErg|ubG(KR2h#!E?B3H4Wu#M-T-CU4T`$-BpK58xu<@~WgCWXnp zVED*JGSX|0T*`tCE5v#kPu^W#VxgJGCr754B3>392R@SoQ19?pe6`IRMARd1KojR1 zKbAMvhGfKO5ayCaE)mr^fNI(1h9=wb8m6C0|?e6%-WHY9;BRGgX zX`uOXH*cIN(&$L!UGdYGgcJz!Q?1f<&rr$Yv}IKdd|{toO#<5Ecmw$9@@aDV8QE0B z2eCtRz7d>&A?o^p`}7HX)<$6bvwu%24int#V8L&Ak%0*2dU9sOIpHg9uOB={8meEqBvKz~=NKQVOL%pMfH;I%KTLIMrLmDs=g!B{AJ&p8nROJ{63o_K!( zwbp zm6-oB0GiV1z59wt!BY3tfgx~v5!QJi1~ylc`VvPKW0UI09lW&(#x_P~#Q(W!{H_=< ztMqa;!mRmC)Ptp62<4qH=#Gc`Ju>hz^8Lj4iBV}8=i+@X|5+pcX;?=1^=|X>&b!;n zS^M#DV?(M|aw-UoF)!Lz>a?>ux~N2GfmX{-eov)^b1#uYrCN>VK3KsccG(>+YeLu`w%eWBHJCk-QD+KYJ z*(Lrh<&~dnf_vj2~!Z;Fnz>$>gk*tTuk zwyjFiv7JuGwr$(CZQHidL4_yZd&YlTcTbJA_g-tRiL^Kc4yK?5WXT?{xToLWqGHmt z7tLZUw0cAXZVfi!4Q8qh`DTto7_D5$2W0b7D5Y7k=|^{}Im#RjHNvsMzZiq6}qDO$=|X#8S=(mS#KN!{^U=zRhe!j6*?ws$8MU8o{COBbZWK zgkzJ!JWOOkhjaPGFC4BXS$fy}IlzENEDd1CRDi0w6eoHN1(^4sECgy$u_22>NHH-V2OOH zUJ@O+)uO4>_J8zHL7-FA@$_4Iao4E*DXS>5mHH}U*nWgIf1*WvLCnf$Lq|`_Unoks z%XydHo@;aimhi-IE@ahC_biDCOuS z%q}!Mm7wz?CKsURhBR^re4R*T7~(8|F`fPFoKlk;qeE=ky|~i&D%byb_!^ukfq5Me zcz^Cb()nc6vw7QR_>3R+oO$++0Vib$_}K3G=yE>6I3R23mT|YM>Fg?O(Sizc!6B|xYd zrQ*?Z?jQj=^^Y-HHLs0iB1E=lhoS#JYG<<%#+_zq9c@8X*u*IUbV!)4%X)zCY#(UP zZx!M<^Q;js=Iz*7`3aoX0K%(X;yr7qp%<2V8c&m)>L3rQ=)>Yq*agb`f6 z-&~o04N=N55j01siSMFC_u6Q`#u96%E40gVo;zQh_1BP2yb*5N_+s#nk@z9_>Ou&O zff6g^Ezc7)1!UYOuUTKg;j)u2a*X~tU6xoHMNBIKP5CJTZTa`NeluT`;=iVE*zb4b z&Zd33wkI#f9~xYIPRQQ365q?Dit%gY&EMa;ZzCS>!Lnu)6BDdLWBF^IETtK~0!+Pl zn;C$oDZHPzNng7H?+76)J?&3G0-&Uk_wCI)q4*B_D#6Db0q|u+;PdmjhaiQa%oCKM zr^@B&hCw@f{xE4>)$ZFuXyos)gtg^Ll*G;i4sWMw#>@3CT@8;wz3buL+6Q;*sKVkD z-EAyV+pHt$Fxp*t9;!5Q!9{A4?>oI`qT0V>z48og4`kJHjIWFE?! z8D&vPcYP2rAOTUP%kTMAAaotVmQPJV^s#Xe26- zg;^yLh<4$Um_k7aFm3>+P?cwTCr@5MWVHIJuW+k}vqZ@Qa+d`q{Q+`%EDsybx8)JE z_I{`+0=ALn{(Mqjt1SK^g%qa6nf)A4z4bIOLM+<`KzbnT6;-+7aMohx42iTR{9ig@D8@`Q* zpH)1x*IM$C7Z}y@5@k3q-}w@SxDs*fBirTm{$v$#u*?Gd$#DUU=b0`j&q0Y1x@SzM z9_t6RIp?0B-jeEaohHO^o}1nVz=CS=o?l~H2%qfz5crl`!xdLW+;(qTTf*E7R29L_ zTZwpo7kt~Fn#27oVB!Vtg5K-!@1UcoLCbLsgDjwU)bkia5P1ENfPl1En5h8Mfcf4K zb3|egS3FUrU!QrkebMczD*6NLp)1}&G_2vsdD9pP-{F0~JR~q*wC{5v??NmkOXa_( z4dQO4@#_knT5@fC{H5{tdh z-L;xKQ=PM)2qxTILr(0xZ`QbD)8-frrdhMxVvuZU%T(+azLM3fuzgTkB0Z(X%1tPU zW-?YJZoMuE*9q$$HW(jK<3$@2Jq+&Kr_hLQuC8ApKKNZoX_VsF+5BD(NpEKgUiU2D zb}=P8pNnSv-?PHb=OM|%Q%R`L+ucG7f!*v=i*ogRX3uc&9sAM!OvE+@?e{k3q33pw zdoi!lRQjXmHHe>M@cdUI*lo^dz}LTyp0|#kiZ9QVgYVa3=Xo!wCFgxC$9fWD4nxuT z{Tt;Vu(VUb-%$8iRG6cvY1c*d+&yus>YBDZdz;9m4t!C9b4~h|7CflZO!Aui8})!X z;N^vA=>x;p!wbH|Ddo;OsGP3g!E3^*@F|HS_`kY==flK-E;UA)oaMtHYsx4WYnG|HLJ(NsVOH8vlcbA2DlJ2Vr zuUod#+L#fexm6t(L_!1jni}O!-+j!JRkP8jfj+dDCrrdWX9SyP&0&X*G_9HiPW39g(P>*!3Z92^abZ%q{WJ*(74MT6cX zkH>~kO+-dhB6HiP&p7}Z1tCybFz5;z`pK}nr<2;RBE$f{TmOYN_Jh#?^7;O4n0Pw} zMTkb$V*TcIR&NERMRbz@MI-k5d;Fg~>tdMg0n(G?84JPbE%pJLT7+EU7!`{%nB>k$ z%%+vka4qfsl{8LZrYDD3fCLJi;Z}Z_eM^GbF7&U1N zBES_W(AvU=^bUubxl*D^BIi~O!=H)#3?l>AqNiCNU%v#<&3}IjE?&EPlm7Bog;;?s zl((A}*z%>LAJFyW$M6c)a~$JEK~|oc9L#!rH5J}@myPk!6FR55{OGxq(hsL2C_mJ8 z$?zW3^Fp_CFF-(^KaO>S)=cnrM_@Op;kCq&F7Q!a^KNRvGsmF%i+$!&2F*|{3$cN2MMcVg^ZN+W`Mme-I)bhBQCl5;U_Dv&dC zTH~S`Una_Me{KLc+g_vQz9`=rI4Vz*%yyI=DP5ReVIQcF5@w;NR_UE(?$~Da3$+=rDZM~nJup5S!>ge!hrLiAcG+H_hZrw1_Pa&yQF8`?v z%NmSyn=ou-m(d}Td*fu+ycL?%yYo`9h22$uLomZOe5n-prCxJ_GVFp-)Vu*!LNrxh zO!m=d6mF(pi5?I?slvYBLZngoU);jkNsRSN3~S=~(a}}HGS0?5iPERNG}^;pbJWz% z-v)g2i~wGGzp?_BJ>tHh#n{4U_$cP3Wm>oan?{xbyVB7MLgk&P_Co|A&N#b8^9fhYiPJ(bT!hN#cR)nX`~&f=21t)xBJ2!lN-Gh zqBpV))Ks#-1!16znag>u6jQ3kqYx87{57_$UEhFp2Igr`$M8^4BD2H9ocCI_}466Fcu8Q)^g@=oBt7Y`T zPifws+Q7VpTmy!trTm-1%tQx3o&NQISU`*iVECJ6!4*MeLSOh!*2E*dRikvP9W(4p z6=FD6peZoE->!3(EUWDY?M#AEur4xNgrCgzP@n=RHdV?LR@ zLdQo$EHe(Vw~6OlM-1A0f0?4ky3-MyG}7_#tSZP0rhc^}*9I#CCXrrbR&-PByYr3u z-YKe|&L#PO+Cm!Htn1?okyk-RM_rwIDcJfvd;vLd9$fGd71eL9`@Wf z6S&j;+PSgp+}GWtAH{uhGXZ(Awu!J5=@M3p#O1O4HTmd=?itr(L{Ff0oH)nVV|$kk ztbPxzf^=9m%D>f?LAJ}1!G-Sv&NhBpbS@E04oq~>)Uf_6GUgv%wr!#AvjWTWYOzH7 zdHlv{;{z8!pzHV{$6IL{WR}8@MmB0BdE>{CsU-;5Xg$iUPgcBwN~$wPQ%9IeMx1L| z{2fX{YdR#496L*dJvSE^J$fh?+M3%N)CGlwQOL%)mk))ZAHgh+ zC{CJRz{>ZCc+8YWl(g$v$~#Ia#+7RLIsit`yhq$g@DrN1?k`@~iKfIGkK%uBepcJ= zp=H!$uGSXM^1PV@QyK##8p9mRvU{jq$hBylx_&1?^eTPtn)-zw4q1QaB?|X1cStr$ z;Si}b@^qQGkH$+R?&*Cfr6`vqOHy4UTDofZuW)msR_gPMKTe4Vx)^$>CM3QyEv+lB ziTRg8oJ^SA^R!#cCYK{PdsSE9fTC7C$|e;0B~T^10{SLFiqe}=0m zOR-foZ({pNO0W~W=Ij1O>LHE`c4t;c6;V7i;Kz3TB?4vbqr*ON;s~00j!Sfa&A#B} zvO3n;e}#ruIhf|kqPI8tBLddgFOT}-4kAuDOCbqg&Rv@-Qg4`FypRu_NEC{2m_X0r z)}3??$qCsm>EU)~kGsIIaJ^}l?p|{!V?Tg9P;CqTS3|gfQNJ{vf~P^W32|JUWUo^G z1Cv75xRwQO*%7^GK*A>h$k+}wliMZ^p5}om^=bCkD zg(B8o9dU8+DJ*}R&Wc?5>`C1{E54{HWRqiD9GWy7xO+-QIGz08ahu#0*Z24ut63$r zH;kSKvTL68sQ8`|cD0I+6kkv4XARaoe2lFBSj}$tyb$Xek%5rAhavMme9CUDN)$R6 zi4_{&iEQ8_*fCR^+Q;f_#fW<>a(Z+f`Ri*h$Ha1J0SSN zdXpScux@`p)LlL1AUK*?E8+i(t%k!0EyZi87k2@2EDz!6kh1MLpEx;B(1^Sk$6(`1RftH>c#EQjT-Q3Suw z{)OP0wBxi*=_qESDgr{XH(M&cjFo_R+*FhP`#a7|O+{O21+~)j4-A&kM(sBjusJZ4 zU;UIMU*XTyFB86SN8#pDS~`Lmb3<_z5Pzhp0#4!N)yF$fTs)praF_ZRT>_EeXw{`V zVsAf;s!H}-x9P=6H`|DF``r;rvr_sJEmXr0 zzopB&SkfP9!LhQSNu)-`rUi)zK7sZWJ|^Mq>hz!1V!0ZTe1ComiNG zhonm>slL>A7@bm*ma(tupKi5O)kiIc$(SYc-4bs5%LnMC*kJ`GA+|w?3Ng2wK6Tb^ zZWpV_I|(Y-NH4u`1o9l$a0D)OXEcfIeS`<9PfiAQ$mE%0NYpT6(O^_E(g!mq^>~HO zd2Zq(TCTVT(&Ng$j`H>-=tneHrZ=T=eWRM5X_Hh(t)UUg`c;d#eHDGWaVdNmd>s!k zgGL;d{WgsBu=NJc*tjgk-bS2ZNMiEI&KRbn#5o4z=4v9XN4`V2OPmAk7tv~PWINK& zyRK0j8_$81kz6`Jv~xt-(_T5~iN#eF!&`aITU`x&v~$tT+bst5n9`kKnb&=B<*1GW zp=)dRA7G@9Avb=)_2?+1SeMs9&8wRJ`$&%_?+9SM<#W&O!^7w6hN#)@ec0~3+2=uK zerV^}!)Iq^7AJc7v&rsbg&%mP5G4dy*9T5~UFUqZe6cQHNqGO0=sxgy!~43wc~2n# z*1U{#@ALFLe0^$srIWr9~lN$w+5e zf)QmVuE9kRus__kE;$OR+&y~z^4WwayKLU_ux#`?59{P*k`sN=a-70Y!~G6!bt``_ zbjAEY^=zA;%;J6K8(h7ZMsiSCHHF6|%!p&IHRt~5dAOaI={-^N zy%t6_c{4oIGM_d7;af2`OQSfwmJVdQjURTiz1jEdah)mW*3oP2RB{{T003mt7uY}J z_}>10TX%yKtSsoiD!d2Rd@_7hJM=X6c*TM9d+T8? zhrY5?F$WNIH5ZoHy&dcXmwEjw(0_eNpzj85Kkps@r+vpL;uZQ#;l^9iP7&!lWJGf`T?T%ye;>N*9hr ziH8e{EmhSQ)!_{yFmglcioNWoKxwGui$<39fJ>-+8e-VT$p3RXn$ugc9Pg(ro6RMg zRS$6ac32DFBGWBB=aw%p^jr78E$#P6G&+ajXcwc#e%7RTJ5v=8OQNG#3k$ki6TT-6 zl4Z{oP!CQ;GQ&nn`8}`^x-#nMyeWtt5__dcJ9mjy-XC{Hgv)lINyurq$cjzs0ihPF+M8)i&LU*P;Ri+&Z1}Q*1t-KWp z1K+!!tbk#5!VzY&;ZX6EVlmSELDhB9jN1(*qT%u; zVk)dSkGe7i@nZc4c7P>?#l}X{+e0qIJ@7)1ux@$|_3V+!?*0MrvDF6E zOHlEy^&&T5WME@4o|K_RK(*vwlp*++#s;Q9z0^(6as~-A?6!>~4L@g%yn$z>cN9DjW$HKb4xN)#JQE>2 ziZQkhV;ZqNDe#R8W_3(O(OwjKPAZx8k(qZLj%&kfRG-oi@(5e%ow(pbyr{En*m^AWzLceI9>TXM96qCG*`=*|5Fr{4WHUR&M`E zu>*`RFQTn2h(d;SJmAMPjqRJ4&oN65u&d|!`Ln9$v8DU0#cLtOXDDUkQNibR!3VhW zG2!zs<`d^Bq!9TB*r|sv?yWxYH`&&YPKm@}{YoU=jTTWBC9s~_tVmv$n5F-Z&TP^?b zLjHmx@D`@otr#^&Ej4Bjl&HQ4_xlk>Ka^yQ_S#L^i zQnUW&A?N{90dFKy1CZfQD~@JECg&O{jk*$(V412DUUn;y!G*feEL-H9p4(-0ao}Ze z_hUL<;^#b_3%T8AB3%4~i(Ft=&DIcr3e>%i#wdxUrak$gOlj(>$l}KU%EdiKZRyNz zc&r>Uml3YH{@Ba(g-E=FiH^QK@?%AWU_c81ElC5QuE5uH?XiH}XpMy!l8kUq>kJ$H zuTPOl$E|EBQzu|DAl>8QahK(nr7EAiD_KG?8FCGH2eehuK%AQPIHCEKGjGP_JRG$4 zjN_G-TctTcYlvcI;KZYdBnv3zDBGYpnfSMLlTBc)V}fXz)Is;D9mT5C1C}xyV%7AU zF@T=;EtOf8$9z`$Y8f&UZNl4hC3mINe)IYMVJr05w|K`(HU*{2p*x}Mb@63aCtQ5b zjbHpao~zAI`869y5EFUbhLyY8O!Ufs4`#+4g#ngy;$hn<7=17>Tw@`8QK*OMY|XYg z0|gBQu(X>UIjIxPZ+wiwj=Y?V{Ur~YS67uUsE*BMUcvMuTtj;ku6l}<-r79|uv@>5 zL~_3Bj*YH-t_&ZxhxOePLMB4)zmsucpYuL;9rTf1T@MMcyk;lZi+1nFOjho5-f}** zDtj{S_h0*~K7j6@;8Gr6cV`McA2*&0E1Fd=OJ7M}+ZtaFUt2a7>llrQi zo3ER%75v8v20g&F$Gf0_lbgcB{?GxL7w%}p#gfG5y%X@qichut24#C2pl0>Uy2SJy zI);m(=cB*oeVpNAKF0eNgy8*q*)m7);&<}}@pVhEf5frYWpBsOlC|;4>G-T;*Y!?9 zNBnq+8wn z&G7XZ_`3^R-<$FzW~K*lMKtdiTIN8tBd`0Og|^?0GY}9Ea^4Rqe4bEpTycGHV#TZQ zcFlSv#c5768Emz}p18BEUKb2|VS?l&?S)R!LRxTcv3=Hu-@+f6 zU0$Ljw+Eqz7_Jo$St1vzKel&!k_ ziz$%ebsegk;;me+@F*o&lG1%Mwnv?JO>_I=CQI!`2HG1Iu0xUlnCRL(PqKn=SHXn3 zGSS2fPJE{eV-blYs|U*q*7O@HZ)ZIGy!lnZ8_)7rG@@|I+t=F#m`?@LX~f0Ufl6Zw zRk-OxUy%2)05FdLc-><_q!w~6@KRUfxg*hSKHB-7Pzcs7^TY(Jw%((wGH=Q{l~QE` zaMd<;gSvX_xp~9)xwYIVq^U~0cc(hvTKhLVV{3_j_G`;cXT#R;llMxh8IIWYb?Og7 z)=#3>VH^a4_g;wYhvN)8pYv~{{Ii^Gj~UMi#yU9hBxQKZ`@4e?jE?;KZ++JO-MGj0 z!*Gbe``-7GL`C4?E=HgmBi!iQcwXm~D>ULYENP%M#SH!U0mh@*dDt;N==<^w%x=5| z5K;E~PD^cIDTp_x9%$PYU0v;I{>Hw@23huS=FNEE4##*JMxkabyI-o!FCiRCUlXLx zSGP3v;RRDx>ZPk|r}m~JU)=ttHw91N&!5=RcA>?EP7FsTyN|XiHUc6IWG177S~6T? ze!%#yoZ?OzQq=tyt+Y}zmT#$Te3;6o5qIv)&e|Tz-div2sWG||BtPC=dlwbh)*ifC zZCE|b!7`?e#FDw%fQUa{InZ6>B+Wt-{zvSN_B#mNru5L%>ZDsQn1hZs$JYb+Z$yLaB{=v zsl{u`776&L=R!x|wZi8@%+W`k0$;>iKRx=U9`Mrh(D8L~1w)ncMLKe%(A{~uBjMH= zNaF#F@qT0|2lV9m4+Bc_G5>KVN2sGc;frsOMRKq`FQ;$5@13YG+|KaKt#hcp62hiU zWfgOK2II-PxjA9~3Fy8X_SpXV%z4;;?tZ?Kc~h;C+{a&V962vSU_o#<%Za{Ue%tnZ z;Ir#KQOR-lth#T=Y5O$mgGXopy=1KSV2;Ro))}09a$H3+-Pc2Dn8aR`S7T_T9F=z6AIO!E!V8z)TkKvsjti_!(?TtxpN=t-5mL}isleH0w9s-n8(Fv-U~VLf!hpFu$2^Siw{jn5!t;g65u&y4Lf z-gB026~Mg9Vn()#2o?I+yJa;2+cA&(myLXFG&H$GwX!o$yCvJAmqTBh>bU z^mw6M1zU%&C#+6-YeNw!CF4FNuh1Xa53M3dRI6&WerH=Ez~K)q$IXr?e9wdW2XV6^ zpr;qi+LpnUXz=>B*b`F^fqR6H&~2wJJCe!gH`!4 zfx{?sEBx`xdCXVoqz%76*I`Jy#oHcmC}XI@c{SdJa!^0FBw#{r3^3#Q83`}{@BNpql$83s3B^^W@ff=?6(vC_5Ewl%x5-mie z?8?~?fhJyvP9=3lR*r_hTU%h$7`{IHiO1y3$P2Sc9XYu`W)K)C(>v;*^i+H|9L@NP z%d3!znjGMsXnG>kY8*LP0D;yKhLOayN^4KVz}}_RGumptOZ5b=@Ew8royw_}Ach_+ zkxDc3NHo-*h{yA2IZ{S0UFS8rqIl+JsC447+&gO0pHz6ur7wz2q8VD@bd=^;H0{+# zy`fj~2e@n4Mk8A)8|z*4K=KsbO-cS2tx<3NkSfMLXxXIytsZ*8PJ4_2^@>CKxa-$h zLS;FsX7Z&vQP>Sg<`K%(d}hBZ;o*LpKQp$KQGCsVn?Z@LHfNo&XX$3~`o<}sLWhmh zGQRHm^qwgNq2T(@_NnSXud992!q8kZnuf%~t0cGpbm-uTBV7^1q) zDpqOy9;NXx&3_7KZQ4)99h$MP$gZ0owUw01)!9=>n&>W49mbmV(#escVu*K2aA>?m%;~Q`fxGcDSm^9*=(oCO}wpRw#&g^&s zr-frPc7rM}xlo{NKR_UE{WGU=y70KMf`yEu)S((>{o7}8n^J1)iInk4T%oWUW=Y0g z#1d2BSjpn~%%&YnWk#4aG(z<;t#YgF!8F8>9jeHO9Z(O_M;SxI<}Vm4WRkgA7dIE3 z)?~s_TrHSqAA~vbVf~3^lpYomZ8-HQtf8vblWIN>Q#c}ta;a}8?6@5Q);geWHa#x+ z_M{W20Z<=(Qcb|^#PVv(lVUFzUs=zx>%P3pF$^MygNZFYJ+DJ>Dk0+}W-%lTCn1tz z5@P!O?6O#}Tx#)po}IYZZ;GR}<_tCp1v*9wy$*I~gOd=aO1)!c@fbN2SDK_|UYJRo zNNcmEPgMOx7rXU3^9L2v#Z)#Qmogk6R=u0pxFSz zIOV|ql4~H6GTy6LQdoVB2x*d-k|~0b+ipT9f(S_kMIA}&msqmmlb)bJ<-zwR7mbUi z&3c=tx>|3mHRPjOys_M^a+{57M-b zB!qCl6Y9`6>&_KvXcg-u?ss(DU<7vVu-hUb18bf`DC#Iy;b;jZC_hmIs>4Sp$suT3 zQ4*T+E`%oQkEry1DQ+4aF*a^*Int2mu-U%VJTu?G#QssqNa0U8Ul{mMUV3Pfl7NQ5 zZ_`clUdh)hUl~@!uz!&C>;wn0 z8fBMZ+(xiHv%;g3A-*PKPmGR_)%kiWVq%hnRc%%?Vu!B$yF}GC4%n-ngR= zjW_?T!3%ZExu6x_*HwF-_KN~R9bzRfv2qpQ>qGIBg2iiJFULtKcg&>g-+xex+6P$p z>F`mEaue>yrg$X_3%ii_R~l31i0Wd_4faRtz@XGePwOQEYg3!r(&8dMz4s_J3-tby z5E*eCKDd|%XGNDXn?VjN%( zbp*nKRFGp=({r`Pny&i;7C=^I%Afb~$?HssXQhZRkA*@xB^nkJj|wX%bPfo+MS=nb z>LNjrde9nHL&hN9Z*sMntzI*)+A6C{kkS>S9(P= z>IsUn%{)tGDgxSdJ6}gnAYyxhCz0oqr#T1OsfT@Ie?p`MgotqsdAMtLPB_60M7wFdhi`wI7=_euIrMh|VWj_}KdKo;!t))XL0t9_yx#8s zZhbhmomD{OB1<_Iubm>q`DQ2cb2XZp8ei}|E}zm1qD5)xJ1*}9K7jM<>dwJ&wJJri zB0htm24-HL4W+~TVi5wsvl;dQ4qJLM_tg`#HE)^nwb}OC4}7nGN}SgW zUH|4~=z&gAkL;K|Q~#u#!zJh@5xH?=;>e{6l}oGW3qkeph>~VWPZRLYKqM%b>Sfuh zuqlZbxx^E3PPx7Ru!q;BidU5i6~fW4#choJ|MKrFNVJ<7-G#8+lpSH3dx)GN8h5WE zqRJ6E!4~)eQ1_glaw6oN7PHR$b%r(Dh()9YT~3#3tQMMa9#;cuCFN8cMYOArn2&49 zvz-oXPSvbCp;YqJ;p%wnz*Ii-es#xf#09aIIUpu4lvXbp0R1$6=UWNsvkFdxO_C{s zFsRyb{XuV9mjHlc=pMulHu;?mT!}9?3>#IcQ`Dq^{jGOl;=~h;lH*YA=Zu7arMKAF zicm+ndZSbeM|@%_C)zXiC=wt7LBpXappa`SdL-?kA&alTmhrmvP&L+e^RFk|{i_;O(qtREiQUh}kDcnz9)vJ2ws` zk1`y?U`zQ|h5ANnc}dfakUGTMTo4@F2$Z8pL76uk&aaCXq4@u>0G0RzmLXw4o8wY# zc6JyFIjv@E>~5xpec@p}?!humT*J|ZUCUM7NGG@slhd&c$F^&NfKUM7l!`d*AlA$^ z#j2^?8p;In?c}z89+7=;AEIs}xilk66 zsV=UE6M;u@Xw5~$;UCc|2nBzn@pIs%c5fVG%|DtHm6L+&2UHW{;(n_+b$~W(li=ng z=^5ijJYbl!FRQ8aUdchD`==EuFjadGxk1?RqH453H$K~K?4BuvLZAPb;up?XMOw}A zZ#r>@+A&3$T5uo1(sQa7df>E&+>>!HD9ty!Lp{roPzEGoKDMy|*M_bfiy1*)8%!i9a`UM$Lnrw;mns$WF0(|@0UPn} zXgYQx%cwG3eK3+fOmIKzr1B4ycqjS6#bYLXm5>6dUCeY*Yzeryk?AN~rul_che+L2 z-9Vw}Rk8+508jKiX_BrGeW_AKr%I$YHWkSTAxs*Yr6pWH=whKy>8k_@nHxIQoRk4B zSlUtOgQS#mtcNqFa0trj;;W{Wg)%#mIH`4j;kEskq3EnL@g4;meIP$?B)P$h_EW2Q zq<;#$?!G7LF0v8o-oA4#qu|Ay0j30g9Q~iN?^I*ySGN;C-Q-_bU=}?o;VJG}(U0;j zyQn5!(+`MLYvPFYb+Gl)hwhKKyqf(zLt3RXmZHE^psL1nsZ^Ho1+TnW57T_}6W;1WvB^WIAp236#(}Fb?%h zd*Rv)rb9#v_F!X`OA&XG8;St_u(23Qf_YRk#Z-7q#_HH;LG%%*wPQ5zP-Rk&1Ge9_ zirp>Zbt1c7NK)`?ri+TgU%hcYztQ#lGZhr;Ic)YKBrCZ}@9F;XON{aRz_evf+`TiJ zAa_f?kZB^JU-Urf$BkcwiP~BO!ixCk#lcGd_KrTLJp7$1J~6Bn_YafU8|vne1ZKFV z&^ni@eY-cUArsjl$|#KzZN*x5eIT$+N3K&u{)&q#&G(^!zNAsw9%Za-8?nN``fxHO z@p2_gl0*zj>zGaor#EwCL-y7KwG!U=vFIPyJxDQu_spj0O=C4dH%Ad!15uZ%SsRp$ zQ5jh(wcHuQTS0qCPvvm(B;kg~=-4uxR$Ke^f$}Zqha!^L)FB{a>!f(V2CbO>&l#6I zk-xa(ARKQ}oYZ)o87t_v%J{BN5x|qv^QE@VNP60=VoDx+cycOAlWGJ`J03*9*9l36!I zIJGD+%0`6f3x+u%qqU7T7SfS99;gL5p>Au(RTp$>x|$mG$~Z3SS`4*(rvfZBFo4D- zk;i(zS;1r7R}zmGUMq|%;;kpuF#UZl~XJ0^)HRK9#`Ld;$I(lR*7d44!9_(Vl_bf#htJ82he|j zc<}ydziY)4s01dtnYcypqgj~g-YP*@5Oq>wnu7}l{od*07ziBgb{Iz~M}61!-i1x8 z^dikR(P2S)e?JMQj_tmSxrQlHVqzG}5l6syO)BySurW&<1;{x9zL_FF$IUWuP|n(X zN8Kp70tf{>*ree%kHQWA4jw1BCl}WIUVW3Aa+Z9{|D9(0hZ9!OE=|69)3wL1qPMO_ z^5`k={WsS(L!OE7&^mLKV?S;XE^I>6<#0{HRVesP-af|Y^gT1PNP}}%>RoxODPgq| zb9EV}#8@l1p3LZ&E5tVJA>~tBs(E9!Jzhaxcsx`Zcjva?awmKG2;Cn(L8>%Mjp0lL zhtSYR=rBT4+RRJ{3oOJ%bg@9ka|LfMf{+XKvX0gwNra%PU~0Xl>lUqm9f`c!A7&y{ zIs!Mw7>iflVTBacJ;rkd&r(Y|MN!&L_sJ!mtgS`|fLi+n{?ZAWL0ei)&edmmC0T)| zw78Gs5V$w>5S}g-YX#Ssy#Jj=F8FC2tbxk3Hp&-0vphn3>jC^xm6atEVfdd;@j3`5ch1-8V<$P3j8u1O=D zC-A=a!aT92rp#8XwNLBHyxN__u!{Nl;045^*?DV$lQ+6D)9&r^NziWgV3{p&UlMl& zE*DTy8@?Yx;A?sO`8=q^#_9i*AL=Y9d6X?}aQ@t~gtIrn6q4E%`;biBIUPK8w+|n& zic;U$;XFy9;92sehJs~9lP~6hr#xYrSSf5s<|=<#fv{%@$|En4)Q5w+}xWxQ|% z$0U$j{z5eQxQ$JW(bAV&<|aKNxv8yQ-0V9c6@PO}K!8$RnUmtYpaWH_vzjVAg!d?h z!r)UcxEGwj__L(>_K^EtdDJMmc&u@If#PqPoIttr(D-58(7nV{4{n|&qB4QF+|(|S z2H2YV4t!|4;F3tret}yhIii)Glv95=wKW83cS)^Ln}H|Y?s{?EV1v#CGn#=BRZrm8 z;z)agPckQcff-^CFMEO;RI5)Aa#6z0gd{mw-Z#^XWIGa zooH=akMkeZnrk|rZ4CxA8c$q50P;oy-|qOkF9>LQ-403FiJ`_&^06Y{4Ln=rJqyj7 zAmV+4cC2-Sih5%0cP-324)PAF8RD7^1%*+ew3>L>*oRkRX9F9G2bM z&hV+BpU#TPh8eMrjYCTu%0l7FA@kUDBZ)GeS6y0}jVfX}sm}_LNdD|=50Ap^5x3_t z^`ZxJO+&RjJc4BK!w$QXM5fAsP4`V7jF>g0%$Tt#6h05)wfr#zj?X>1uN;z?DYHlf zb_!WYX@QGRZPbsMf~f`QJXwLSv6jK(K9o?o)eQs~CgRGWaRdD%ITvIKK*Fp*%Lj+x zWc(tIng=*X@X)~>n2=NcyIl3d(Os$SL`0Q?U*z3vb=g#?B^;T8lyM(IdToY7*9ww| zILMPIRKicXGGpY3Tk5=-t0MrBzJRR~?PwH7`P)Da7r>zN$$%O#`ukoz^t6`x-*8FI zsvOD>rB)uQ#UJBHZopvV5ROxKNz&RNHR>w2{)@&WjS$7D9sul5k|uDATlr_|C~i7W z)5dKVcuzHAlby?Hj3n+b$_=11PGm!t^%9-ILf|cNb%8X&j%B3M;dg3Jj?LkjiB*(H zrK51AHzP^<(y^U~YbqM5e?S9z!QT*oj{Gd-JS@sX*PM#&F12 zX=@;mcKYuAX;s*n7n5$$>Ix!dL*~U39myPButt~<)#!GmC?~Q;M(+m!RA>hP-D3LT zwmKw3TvtrPy{Z3IAPS1peNW1WcHVKhm_vo`j3V#Jk#kLXHby9?TqtGs<6tJ^MjW~@ zr5Rln#APOx*&D8{!5h!|C#@;rqMk%~OIC zQaR`@nsTAJRvgtuqaKOJqF!7o@A9xA`BbX3`A=GbI1WsmD@U$ckbD#^RB93(7RWm$ zXlsnDs3p0srMm36Z`%4#VKP>?SjGWfy9va2lyicjknF|YmkVo%bnb5`j_^Mz!fbJ^ zQTlkdUPLYM--PcLvSUWl*ec=`GKS^$#!NdEeM*n8eb^8~Ea(Bud*^NGVzFVhELxYj zPwkcKg6!y32fMFqx#a4~h-}iu8?ak+C}<%V#3qN~KMHVx&>S)fBt^WwN|FwerK2%5 zm{jqngOX*yY={_KpB_z zFOsn6q3<`{Yt@`#iYn6Hs}F<3A!+rjhfi0M4~N#w4}WklU(3&sspH)g5p?+$-rXsv9 zSvj#`=)B0jeJ%>@)w;xjydDp^1Y@CT6nxSEs2>|o*KEe^I*7vK*)`(U^G|| z+D<%wyV=w?BtUDTREN?ESCU**4xo{mI2s4S_A*LYIfx{pQ2FEJ=l$Hyl_lNyL|9X$ z-KbP^WnGl*KF zr1*_3j-|?U;BEwZW=KwCXY9sR!ftvL#?rmHIn*jUvbl2>;Ra_u$qd&}fRYh&8KjGb z{Rn7aIq3WTrCl?a<;W9IAw4~&XqPP#>GVjh&f7b{nP%K6Vs~Ja^MBQ z)y2CkUFA*`qf_M`jOjk}WXxS4Nwm*Kl<&HX;&ieQXfuc|Luy>xzKKI^abH!>6+pnt zCzgZCLq0H`zXiI^ZOdU^k+&+sF1kkZL7W+=kWsPNF5u2Fl$`9&5mdQ(fZZ$GhoM^k zwl{k#lmJC7lg{Z-wY_~bk-|W-#C)eZ!s@*lXfO~=r^%V4PVx8;R>vrc4TD~LwZ)PQ zE^(RC<9FJ%A3B;#=Wu!gI)iqrCE^i6>h~PIv#`)$J}2zX6pw=e`<9`VX8)+=DtN;5 z!2F_5h2m<-%(#(97=y`VxK?ihZpLqjPU_zg&#|>P8)bUgF%4eD7s(a#aWThJ3Jm`3p`;Uu{X`CKhqK%)c=XHDl!QRvArL(R*7pirzMQ)?L~8{qeYx(QXaAr^t47 zT&^6U?|zB@AHa#$)iu2&k}W}Wgv;kX`T2I!@a6qx+j7O8Kwi}}oNh(0*Xg?5X?ET5 z+&xjVY~K0MxWcJz+ylGzUS@QH&9nvtQyJH|F-3&(8 zMt6uv4j4Up(JrUh&>V7K^ zx5}W=hcx27ca^i-Ogg*zb9`4HYmrmD!SqKd+z6`os4d z7TqiB*(>LEaIz|ri#s?)Abbz#-)y%Mc>ln?Ritff-WGn6g+9c5HSn^S&X>IEj3d15 z*``qY_7ns(b-F#7%4K*_<|hOYhwWX zR)0vsy0@I?t#vyT_fHn`Y^k6Wzt0<|T&@?v0Rvo-i$hSXB(K${v2<=ZCVmrCKbua1 z@-@8(0kRb!e@;Ara^L#UV9q_jJXM;N<3qOGGU`cLNpDsCR_&{p*V9*ONAz*8)D8l> zWCxUN#wd92Xj_MeAj-QHmd!GrG%3uPTD+3;RhK7KX)$%~qv z0aJ3bDs39xM00sUw*z4y_fjzp;j9QW<$6f-t35~I{KGS7%(uNY4o{a;#_B3-rn@B0 z%;dFYZ8H$wXXQznUtxx&uf!&Qj|mNEj;VS0^m-|T=klaRT!|$EEe><;m6xn*yBI3H ziqry`8;_QfkOqdOE`Cv)#v_A1KYa6%A~2jR{gxG5^?cRI_*cH##9So~^B+l%B|2%Z zMSg`RzO?ZJ58Cz{j3b$msTupKw z?E^=XoRDMAh{$AwA$eBMPM#=DD8wh<({`i3BxkPyk<`bwwK#(p{JU8+hluBiji$_^ISMAsdFws>ELtxFp;9ip4tQF`-&zQ=eiAA-$+!(_*jfgdHV7r zby8=Hd49J4TE>!d|2{tuej2xJ_KZw0Zw+~vdbGx#eLbZzxl>R-{p&Oq>wBIF@Y`Jp zWBeWPH^R}|XYMX_AK*)7^=2y)YYcyn^}#NBP>b*W_>(KIoXLBX3Ah2azaO+F(HIwY zhRIxFrH|m{MKEGA`r1c(r5EHg^*bn5= zJ$w~XciRP;2;XhDK0Vc01RYHR9N~)%HSf1%O84ryW_s z;?>I3#J$3_=d}iz{8@z5RXJ-6cn)U!3RE%J~4Lf4fV5jNnBl7OuXhqra9e!JR8p z`0%7C!|tjO2m*l(-H)i*?F3ARD;vbtw^WL-!>;)0___HJ;1FQ`RvhH`oBSMcA)c> zFe>qzddEz^F%?jY&D0yDwrxdCUk1{bPr~Iy5`Q%5>hFCZEV<-|T`c2V>yK*`*Oefc zieZE|^hOON{8aq}@xLZEzB<;98cq8+UWa|g5jA?!27aVNQhM{9Us>HCx}VS}=6Gg%NpJk_nUv)=?$wKrSiga7?JVmOU^F;G@`hVUK1wIzI zRq&rc;hLRi8ixm148$NdF?S=jt3IUT0-9Re{_d7&DY~8cjif$!Cp-7G@xQ_kFRJT{o2xTj=r1*Wn1wpp z_njy)NsS%Rjo;r7g<4qDGxW-cfgT7vqMf?ji-CF@|GOQ>F5DAwe}rhMbSZC#tCFpX z)L}uOR^$)!exj{2Km>SXfO0PHjqZD8B@~f9R{LB3GTksYa^b79&oi^jo+tMj0EyVe z=h6HY#_NI-PGsFWRWKn|4s8`@91*b$F%vf54(&LFT=!8w6i?A2H+VBXWfeI&XZ+Hm zxy8zVRU!%~^chsj|0oOK-!QFb3&+``FBfr=?KiO2a?Tk;zuv70^AbfvTHYFgdiRBb z<{JEfmELzFuR;;!rxt@gd1{PrUDk&*%e&5=vnMm`yNwhVD88;Oxi1K#uD<1y3Mo-(+LdSeF9D6Qp+ZYO z$m=_{d=Q^Sf8SLKYbrg}?CEr|=2_=C`x!%ti<&r~F|5nXh6|Q+)syZhf_qg~n1_dvzhJ~XR+qYDa9C0s5acPwkMM<4w{IDb#h zuet>4&t*M0xI*cwo&T2jV3ULodQ7L_vH|f5&aTxlfTvV0+xJ)?=2cfctZLR!csw7+4i2Kk|G^ zQ|ZcWBAI(-@u-1zSU~XeRw>k=_x<|wL8uXh3y2Q=V~>UAvLM=39KAuHBo=;VOffaC z)HHC!a#OZTC2;vX%cLr$Nj>CW2%XWnlbZ=;)djA2loUh^Cv+X!M$t94ex+^4Czs!7 zb9uK+qQ7i=gvvGrt+I6}yK!79tUxG2tm|u^9GBfcDI;o{*rhd@^04n_veci8gX2ZR zSNhdVx6J#a8Kx=R2M*fIH@R<`0P?vqB2z7qeOkhMaDWdH* z^iiG}I1h5h9=G5ouiLC)TMKS`L)a8lk0z`{+lPdfymXSg#R7bDn#6pr&x-JbuKvr= znH*lrMYoH~%Q)XN&Ks=0HM$vf(9WrC=(*C1aA+MxEMZb(1m~+gPq$|1n~_UC5P|QN zQ9KuueRL|u=6`8rRrBRy7Az*e5WvNh4|B$3Fb!?e=D!)Fr0=W~osB`mXAG6{dc`aH zBIyO|Bo_JDs}(HF5T3ERO;GGz`-bEt$^MTEc(7#4gnZ+@wk`yN?K?yk=gBCG?pI1t zYzC2u@HP^@kq^L)f2aGBUqTbVcT_BKG;y)`Tt#%hNv0;?ov&xTt{EyO(g-)CO?COS zMcT^LCU#8hgr#{gIdVD+dN2dWq?n3}|7;RI|E=wdcg-n9emVNFSJR*AwJGnns>cjE z9(1&8B|L@zZoDrMDkr+ic2wfx;toh{s?NW|yK!tdD=yF#5MEags#(Rk=fK1f_d{5@b@E>`_z8g)wJ(}g?c>IIy8 z&**fhncmB9>e(hp3m@|Gv!|s~D5VC~dsg?72nu@?_V_|c)x3>R@oK40Sy&7@^>#T$OtdZNGdwGxE)tk#O>&(L)(Pi?;yJNb;|M zjGnLP{}}G8SBJQzn=n;mOE)Az6U#XV&L1im5ixXd8@D9yLRb#J$ zqN+0eYgy@>5IlNPg1f`7l5>3nk7Xn6MF&4Z^{-K{&7`9rg7mGvsY;kIopyTH85f7@ zkT{207&rxd=}mZQ8J#MVnrB7xF?f3Qn8%j#GsXZ+wqVJYniL-WGNbDu&? zXOh#qKRtyDen#Tr*?lTmAdkhSoKScBn^FU|Mnp` zi+6$hBs<_m#e{RY8L0H>;7VlxxnAZy&~12t3j2QeLT=cdY!LUr zY9E}}_RAm!)+*KDb5~1CVn~2-_l`Wrj`ZZKm3&{#|FqQPSW6~~2puiJj72nmx_3bN zi?Ga>=?dfU3h~-_4#0yDdSB)6xC8YybDE*SCJtc;k*;&)rwZ;*>F*B$egIW%8ls@< z;fi;p20vBM&2rX+JS7%xsDffv#)hu1Nu&$ix0^49k#;?{*Ql1fmJq|s3drkZ!v>Rs zaDnN~WvF3df-m5{r3*SM8G9;PEMvd)g(J)vf&Su*g=9)X&TffbbKWWs)sUP}-<$ibdr zV`v2~Hhch0okVn$Oyc@rTImC0X zCam1YeaUT{u={)O%^taC`{p&-WA>(ViV0^Yc&|=LP(WCiP#ou!?y%;;| z3MXfgH#(b<)Gj0gyj^7l&e>nEWyH`3T;zrU&QT*7sK$Q|6T?`E-+y*sJ_qw|*micp z+ULgZ8}mNREr(nU52dVrJ$WJuvkPNz=lRff0~5V* zK8ZQsCj3S@HTzF?9pC#cWfl|0q61NfoNdXnZA^YFXpx~2DQLCOwBNY6y&})}+J=j# zs`kKJ^xzyyTI|T@zce{x;3wR^HRvO92iT2olhLyx&{Hc+QN9|eqUVDjBXA^-UOHKC zKxyb4eVC^H_8}NNZz}zNM3gWmbUpx%be6a5jv&wH!Gtjd7l^#ZC*|M`kYzA4nT?a2 z>wj~`mwySjhadMx;ZMu78DQf^K*{U7XhIRGB<{c?TTAS+bUd^E#~gGKvw+TWl&n8* z-k3%#c^>PR54B0A@JS_p>sI#nHy#gP;mLKgU1hTZeHlf? z-LmcI@4ucPkDc>@j0z$&{JeMck-lTXlKN1KPt0+Q0y_jC#%o#Pu`#q0_cMluAH(e^ zEDY<^9}9TEcnIhR6ic*YmoBT?j?0QSN9J7P7Cn&mGZW@r!@fYTL1hA_5rsKp{F8>q zDBF(@W!9pY$M3%3S_!^+zjV`**C^C1552}982nCc$3$<0DDBjzGmbZaS(L~I&!bg3 zhbHF?8bk%W(|adhs15H%;JD6m2==i@8#U(r?dW#27WTQg|JoOSpVJk2%#ugWC;^5e zN{g?~*-<>Y&@_8#&XR|`H4?HuQP*&ONsQ{yS4Z@JFGBeu%VkLaUokI#faWvmY0?p@ zj=Q`e^`R|Zfn7Z7awJ~048xtkgq(m@9Aj~Dx&v#k%ZrT9?%a>oCQ51sr@NbKKAGa< zpkLg4**?doB<6%Y7CwIzN>r_#l=&o>uh^=Zie*OI!Xy?fJLK7BR#3fB}h+^2qs>TJBU18DCXw!{$3dRIGCW-tqS)Y|o_GWTC?VprJP5uqiDKOBl9Uni?b?EkC?HF72V!*r+pUM{$!?VnI={hn4pA(x^%w2rFvI0{!hqSzfB>I5Rc3LXB= zjHCi_tf!7q%ep0iEpbDR=VvsI9xwYt4~Wu3jHPu`0Ldo?|76YhA|!^id-X7&Al}s6 z;vL?q!597M-s*3kh(+K03vTBO_j~yya@M*|k&WR2OM24LTS`s}XMSf^<_zW*qw-Jo zoND6n2ds&@_b_F{51zf|b@>NKDEvn&-6b6!Os}u4>!`Boj@RU z1YbUDGn;6$3Chc{SFjx%Gp+Flgh=u9m){H<_DVWD2$5F6@DWp!+}1ZDR>7(0vsU~P zZPhfaX;n9>{9(1$QhPi*OH)}63dU1tQfv;P( zR@NS9Af;LlT4ut@St1vc8Tt2;8`~ZG1QC+Qhe5qiP)(`RU;QOZHOPZ6NQ=p{X-F)8 z;|E$n-`Qz?_cUJ9n4L*&fSYg4+VRMwE4mdG%dzWt*e3)xKVF{(^c&YXt?U)+Wl)@C zXP=y#$7y7c?%G#RoPN*HAfugh-Lq9$@Ih*G=`}4U$TQBnBN$Y~aCO8Kd(69}f|;;I zcc9ls0>rS%sdJAVfN<$+fOy$szTyHPo*SkR@6kv~PR6F2gUZH50=lS4&&;IV8v zE_QwDhh2!ErK%gQ0s=9-o8s4}l|smO-ltna?PrVXMOA1(X2TKNC8N` z$JL$)gad#6`B?=JR0x4=;P7ZN>P5m4y4Dw)RWmx>ONcq{=?Gg%%=JRxsvX85A(u_#2iRqLt* z4!^0JT=Ki}&pbyWuW|@1(@p#4*4o(BdGTRr-YmB5l7Vq-(S6a48yCXre{oU>^qM;; z&b+{=_-~#--mf2a$9aCK`Seq$qPg#vDEjK}&=O|J8uFdazTjVmsK)`j&0T$$WoPqQnafawO*l{c!HmD={ zyiauy3^W&KUMM*Fx)|WJ+}S&BE#X1slN3`jJ9xgJYrJstGpGRpDo#@NJ_(^Vq({2` zr9*yj81tV;C~F#@sy)XC@Z(Qdfrt(6;x_xZc8tXFfcfS~R{7`iKVkUE{I-6F)~fnx zz4PrH{3)2h-Fr@d$z_LK;yV2=H+%;>Iv+0|I@x+1Pdl9i9}O{NH(UyxF}>b5#=N zb;sh+W!_q2CgtY++}C97`|x!afwlgG94iswiixLH*~_$%fQ;wUUk4uY^ySlM+$;78 zcfva&`l4Mx%CWGn@$L!9r$bBQ#hX=6SfYOAK3A(q2yB1);f>-R=kHf?W7r`pXL_JX zR+M@4f>Mf^!|U~CnrM<&d2Ej%yH-{K2{F-C{0o(GccVkO>}AY9g6jAbg+s{mx=R;N z|L1)0c{G;LElkGjGxMy;?_Fm%ZMs<0hpwda*B|1?TaRT&jxAfn9bJx|Xcb!Gv@js9 zWXMMKv(GLujH{&qni?wzzt9hV?4{WZ8beybdGm>kN!=2RVwqUAr9$>uk86N14w|}K zWAC3!y`hlSewXo6k0hDPVomsy1L}vgRhQj8i}26_3qXAqRLfn|h?vGYd2hV5Cjm^{ zyjM_t`E2kdzf#4Y!6~^!&QL0LY1`mq;-TVi4DlbSJ|_BP)R}|157qkihCg(X7d|xo zeUYjA?n6;@EZsZ4*T@ezHG&`W!+l;;uwv`6da-W-`1h=$(-)b+SYs-m@+#jEx}edt zSJZbEJlXHUx=IY@t(=K`LkQ*#Pd0HvX++o7F|no2Oe)`7LVsl<6XI`hrsja_GroO#vhB}iuy+SZX`DkJqKHd}Fi5qy-6T)h-S1|pz$D`z&=TFa= z>%8$BXb>{lm~mO+G}K{J(igS9kNJa4k!A|pxGU)=F!+g!Wh8f7OM&vDzGgQ#USn?oT;-@3ua9e|~t=x(p^0ce)9@KmL3e z>zeHyIKW0RxHFG2>Fc>b{kvvPaXnBUGy7}~h*TPH#L^q(ON z->&2P&QhXiXg^zq18xlfI*I7Fri3p!U8fYc?O#6txUV~c{2bi6^^(|b&_ZyH_LSr0 zYb@jSJjF~}lT{)2-+yO<3+9k(VDn+4f9v+I)frm|?242<`_T>+?HF!q9CWBqx`K3Q z+Vr#b^(D~z9PhY5iz}?HSG&gikfyynY1jrp=m3V2Fkj|0VpG;x6GdL^L@sTFw4w1~ zR6ddfro^bm%3U+Y`h)plojENETnm3R|08Uv>#;i;@R z@ry%KhzKE+dXRbYFNq?`A;;^)zH!z8=>F5l_{XJpE<&CJaMX?l`ArR_OhLlvK^8Tt zr)vOTTU_L{K9_7<`$12m#Kqy=)0aiG{`+F3Evv`FuWLWKkkj|Jv5JQ=KWmoWIAFa!unkv`ZxA^8!_!&>nA`D0}(oj7=}k#l<9nuy2g3DtxRc)a~MC2R%3 ze&6p%(YSRFqOC(Z9F0Q3&(ET1^x-RK?B_o2i|i*n^Fd3n@lmfB9jP6^U zAi_z)zGf=bOEiduUEZC!*~C9!tT6NSwRS$bF7`kC3pdd6!_1Ud!FGPxM)?(qV;Al^ z5_mHEne?`?%x2fnxEDt|EJme|aGhkJ^9-cHvL@`TAsukF90t0|2P}(c*tZNSiypjp z-5kloPG`4aD*3ykya^7WD^!@|U*rr5;frGMxY`UvW}nI(q5c)AlogXGuMfGYYJ?juYMGo8G;Zw0E(8K1M4pDLT6W5i1fn}u^x*fwyXwf>P%IJA zUn>F3-Er%ESn3-f8FZ+Qm&|-%{6%r9)RS7XN4Br&pV845yVt3Wgg7$S4E@BJxooNs zZTuUdSL2wwHStMqK(L>+274=;-a76hM?u#fPl$%kjNKKZ8;Z z5*j04XR#+{OLi$(;gV6e8A9K^m~dvZQ)Aof2tB(W@dW3byz-1iwN-Js^KFLI7WE(Ldf#PVkl(No?4$@`r^nS@Yy?KXBRIcaK6Jfc` z08d66=JEeg(SrqEf&pJi<2mxzqUDHKYvZ#l-v9$L<7+p94Xm8Is8wS#TI8Y@$@w^p znXiag-*4s(0iEBqs@k$aub|t3wR`8^s$C)x#{%qiBORaqm3&aHpjDqv`X(Qt%-#E` zAg8P1edppj@Gd1yw_(XiHn1USBh_{7hkvlA(~y z4{=RUMXsIf55u~7mpz&Z^UkECF_HJ;{i-fhmJrGBy1QED-|5?5tr#@CDLo!jasUS? zZj^o2FLbUaRsDQjY20+P?Qhgd3lr+??6~JJ<96Fp1fkR| ze$ZGpSGN*687bT6gKl@#kjj@oBjG2LJnpFbK9!@gLfFXr1rb`v(GEi0i(gH&8#e3a z4M?}zWNAwKy3k-+C|^EC7wSn~kKvszQVuV@nJ?$@IZX&Fxl%VFP{CZM+=9{z#R0ph zaG_bJ3-ek=?=JgS&*>;{TJ|?dh30#Zr|WzD*T{44L%h!;x>?U!?p3JZtVuo$rYVH9 zqaKcAyd-c?A-|dmP3trGT-Hm}WH7Wm28Sb5`gWS9;{;R^<>S~y@N_Y@ASLzpRyA5V zi~mvqLmD4^%&HmTsZYgzoKB(^v@$rXTq>ogJ)D7osnl@F03D9`pFV>L2%ngy;i@z5lUr{} z-%EcGLGze(lTu@l=*o!Ki=veABgc4#A8_h#8#|g6twukCYe8%@h^u=~Ij{9;YBS!t zm|^h1Z$O80yf(LY4hZfVE93L9!0@-B^Q$BUAi1K1?d)2rY&_u8&De9Gcrm)C&*-Z#E0owA)1i95hngu_oVD@WUQQo>4AK$nOc zPC_PQZmAc1rAz5D!2BR7sv@$g{h$R6!d{xDK0IkW-*hER_Gan=IuTvsuvHYYeXWw58uQOJYmvE@-(vlG7a>oaE}_1nCoE`8EH0(qe$&*~t`d(yLgu zufzp+j2>}(CXMwjNUWs}XL^3}lYGW>{ccoGIIT%=Ez2|auAh%>qyOoB(v;X{<&%y! zAeHVDxQ|SJE<($a=E&do=f43N$n*@46uFa8sT%&gP)Uldt0}HZZ;z(Lt>rW_8xkUI z_xI?ra@p#9Uc~r!a_ey`;hWIrdlv7jy?KTor2FEu8Z4#n90(u0Y@*bE8J!X+1KWQY zZZ8&;Cz8|nx`VdPI3u)4>FIq7Ue^yp-|ncg1#{kw^4)&@$%NaCDzuir>-k9KW!P~* ztqAx-Q!A52AeW-0Ss$ywE9~r*M3@+W;w9himXh7-!zV|E=>BRk#Rt<0U%_Y=4j&WK z5N$NORBWf&0SA;^yJX~7+e258yP~F57J+&XcM?m3=&EvW=YHjVXDvy2@RA)2b;hvp z0n+d52JxA+jqz1-wa3Q~B8qQkXb8X2V|zEVU#ti3WLx@D0}2Y34?NDwG}z_Ui5e-V z@$f1=rVK!2PU}97kX79PI|-!UD`r9@29r2_7A6s9?mmon(a_caM$Emt;B>71kCz9> z+A>iLHnFu+{VU(_ZLhwuK&6$=f>(YAq3HWRh6>*a>7*WHi_GBZQ4cjpvyxkjIZwviLFF(vkXf|fusH^XRHi)n+eRb}E&2&`0hitU; z2AefsO^q%aQuFwGV-n}c;<7|4>FusxYMjPrF(#2c8}Gvwc?#8CP)# z3WZ41TL9E%jdVP0cH?IJmxBP>hJwE5hnnSDi-IR;AGFW%x?9m$`bkf;y+_lvG5_U- zw&tzFhINCPn-*PXN8l3uOW&6C@b<;%EH78sB`g^@waUK5K?4(MV zN>uNVbhlgatYllm2@epmjY+8v7IEKeGIO*#n1tpvyY{uR9urC%2K2&%v|l#j(~<|t zY%Li66`?8r-$Kw7s`$QRN)yYF22Ylu5m}QZ2 z51%oapQaIRX1@~K?X6vTwd{}AMh|Hkk^3Xl-sX>A>FoMQF{Yo*){zQ%j{7Wurm3$G z&y+P$Lk|~zJr*SyEYrWN`mo~xo7u3J08sK-L`ry$d7 zCj?yAd9kZ+FDtiHMleX2W;nv;)(4Z=l{3Fy2|33^6vI{gr@|Ur&ZaZUBw8-YY_keO zywDZ8aY&!>c7%OnrH#BL`_W8Y+r<=Q9u&6!`sLX^Tcy^(`OfyVzV&9sOBgPa6K_-b z%NLwhygAl>J=W^(H8*@XcBUtAT$Hip0N)*J_13N^h!@w1w;F5PcWU0Q-!95{YU^8m zxX52&9mOpO^ajP0A=#D12sFPsWwTNihs_W2MZ6b6lGyr3p~P-wF?@6bLg@ImlG<~H z=+;^-`k=)T_UtX<$1ld{I_!eGyhRE(iTI2?YB{uFAyDm?08wu)H%?{F1 zY>&Pxz2FG{l@&s}W)YJ7A*SalXW%!zi^1_F>6TDn66@TjTM(HtwdY@5Px_dWR8Qo} zWFl)7SYO9HH@kOq|JOt30{&<_6|1*=uuwV%Yc%?F!Rg& zy#P|qQ035OoswZ|28)h8l3?F-0WSgxT}6;kv*^|CCFdu14ga?_R?}o{f65>iZfjp2 z?pNa4qlpL0Y_hdIO-8pp5C%%|0nhj!Bfm~VGpHZyrtGa(i;%zqnSrdE1CAdUAGb=8 z)D|#ZrhJRFFqW8>9rJlw!C?!y1CqIXN>)M6#IKWpRb79ew4PthQ>VkL`zRPcM8{=2 zclq?#`OA<$7-zAdq1au`k4_PBkmHNmb+U9D)?nwf zpX64nOs!?#dz)bRQQS`6`&rgQsXzQ2#g($d#O1@l1Ag3K>6vTjqGvhDd=sa^(&T*j z>(q`*{X#4lIrjHvnHA-5QWSS_IM|sGU=TaZl=L{J!L@LYC9pNeceW^v!uQQ*ecvN= znb7wV7l!K%GmBD|1pZ&E@F97k@BhaIm{Iq|w7T?%P;nbnLp^$hVEE?gtPOw7eyZIs z_j80ilxrMo_8UZ(Jl6tagyyb>hT*#6t&@uGqI;V1#uzYRZ|fYV=GN2Md+mG3k~8S$ zOSWf0LPsS?$yO4K@95eXu9e?n+&aAeSDrNo!lqrc zoLXE!5B2x6VpEjTm5oZ9gh^2~nctZd2od+nn#s}m<~dAQ!obdj%NTZ*+!!LzTBr!V z+SEox(+YUc*L`U~en+vyfffnAFNc;RTAL4R8+rTZM}+pmy$Q7(>qkZ<8mDD9b!WCX zA(*@OV3lm6@zM^)m`ktjHw*#hF`P#rcH!D={}4zOpcWsq@{gWR8}G zZTR3d;8l0;P1|+s>Rd>ap5s?ax?A00XB9iU<3Dm76sN+vL&@}~kn(5O93JXj^m1yN zg98cq(?4?!CrT%OScY8hRiZ>`P{Sij;Jh)dNB@f9oF|TBF>y1R^Nis*g>0$T^%M;(;+oJH(LeX}Meuf4F zp`D}EK+J}~`Y*!U&IBo$p=<2X5ST0DHi1@RoBJDy^gLsMewMBJU3H;_b{uvQ}qNLYdI zdQcf>va#C_`k-KgAT?ON`Fn)&-@uT8bm{?t`#=76r#TKZM}#-r<&o4!?^I)~Ykcj6 zm3a(*8rM_1++Cn+xvT0NY2Egv@1&wnRq!F_ep8=RT%e#E>2|ncCDFOA&G(NnS85N$ zY`Ncn>aiN66`agrsyMEgKLK%DJs<2uUsLe+>oAgxr_c-^X;>}Src8A?Te!W?D6e@3 zH!|MWYVv<=;I6%I0XyqD4s@VEPDTk2d% z*?X*p3v+}&s=ojA z8GnRRjZC#u5NYwHak&uhn_q%{wSw-=RAYfAghi(_?iy!rHsW~B1tx$;C98cfVtqHnLo!>cJV#!-(;}nZu??+zl6-(m%Z#Ve>K?`8j291FDE`{Dvm~H z$y|^RaVWb4vpsU%PFiD*aDg7bNZ!Q1Gk6A6uPpG4hN@?Le$daVHqGfGm|vY*#G6Mi z$H$`mLN`u3j=ZqBCmi}>szX1DoUPFWrC~ir2=-4LnTG{ub_{s(O#IKt*;P1_C%X|9 zt}wJTmmIh0dAPoeh*^F5SeiR|T{D^`+(TJVB8;5AEq(p(B`!@?nCCMaGveVYi$J+j zE6J_@Xs=f>E-s_FS8vj20J8=->FJ5kJ1QO?d>^IGUyaME)BH0lxxfCN^>iXeoqmnU z>5eUVb1J9$v}FhF28l-1FLIa9hrb?XXC4h6JBKQK<&(lCuL**`(({pKl}ck?*f8hy zkbpaA3+dVCmL$JM&Iok4Nb`6ng^3Re=c=D>drM^zPEweJ;g!sr8Qgv(fSGB zN(Wu`8U@?;o!{DjWl1h21z-G@<#-%D{nGHQw{@>^`M+wlr+bj8hI`GUln4HYagxg} zSEQ>x>e04hEqX3RB-$=6O|_cyiOB0NydQnBqb>+;>-$e+W@q(A9soM*d~iUp{s=jU zn>_gocmqc20|2?C7SjyD~{@lZaYeXF({(iT1c zbQs`$n0v|SwceQ@>t|SF{?03_lpsvs5P)a+JKx@UTL{1XJX?vWzdw>(o)H_Gq>L~L z+9nWsb1vofGXx|VgheWjej}{Q=)0AmUQ_`mr~`mY06Wm2dfE7F6HCB0_!i9EyQ#U;AE^=Ikd;yujbv_b6_X5XOEdb_2Z2We5uP3E^@Tco`B zdsbH%6P%O^(NPAP9oIe{c){z~feTIZ;SHU5(?T&OH}K(iDm21^#qyCuT1lhFhSRyZ z14&Q>GfmvaeOJK69HGPXB2Y!b7e%Q^J;niIgsmOh@|>@aOTurCWnJ4nLifb75)dtI zR-sk!qayyX^TQDYzIp8l0^s+rd?rKWjEvgoYFZsxV>4Ao==&WpL#<`Hf&|O2R^uH( zCx{bjo>}j5Ze`F~#6Ry=%dtlIWsuhiN+tW<_wvey<)_JZMmm~nd=X5pyDCqf=ufl; zhHGHHcSVQfyS^s48(33%f%?#7qR)Ny&IEWrhQhBu$Z%}XEx!MJUk~OR(jo%73Qk~= zldD}wy;l0wrlk>XzX2^A{`TSDq&jO%=AXC%&0vm*u_yh}H0pa68Yk%;!+h-WB0&36 z8!B$yyef$5p8o4oMb_!e0!5uV3+IsY|9aI3@epY5X4Y97ak}%tT6_dmoo};y93dwH z5VpC8%MA6kGH0u-tlxM|DX;ViV({AW188JWHO}P?IP!RLq=f|MM)P-<%|VaNLKMZ} zNPJf*RyXwRaP==HEvi$y998AfZqr}ix++4IKw2Y4s&CZ=-r5+5eqZ&Z0?WNgQz2C= zfSLx76G!Wqt(!fv)~>g+d}ic7nsR=UbVE||tXx;rU9h`nl z!SLlJ-P@NxLh7I0jZw0E|IsXl(b_3^`$a0mUnyn@U3mvTtF>A6!dmC2|6k+jU(1gL0hy#mUo_^Km#cN*WCnO>SJ5l1}Z9gjB*+U zJGo&thll)Xn_^Yc?B-6(02pGK=;{?|G470ft>?$qS&2N{@ZrpJ3VC>Qi9eg2f*U*LXeW^p`J~9kx)YR33P)9_jkGoO|>dDb{#Z|JY8HG(1Htu49u8yykja z4{F8-i`yW z0=g<=iIdz(qmYY!fjbPs;_-pTnde0wo}@PfjEk9*tX%PeUkq{@`z;VwuPwILIoc+j zG4z^@nNj1v7-oed-7mA0{rb24{QPPe{kA-)ugq{Rt;;@B#y19si11lpt$$FZ^QFMAC{n6*38_&g0FhGp$Od;4<_v*Rr_HGqF?ub3xU>l~iF}r4r z;-3!$>NI^DZ-3(S#m*47*$6Y-gGSQ}n{AY(#3%C&2-bdVEe8kdpw=ry7TJ(V=5ls{TlfkOheoZ>$%pM`}M=$xFU>)MoYm46@=4L;uX^iK66u!C$&ZB5pa*5KnUXnA7wXa$7 zR;)Z}v)-hX6(bCdI~Rn^VfIJ=+An7_-C_clrFp@GY{7rwBj<5ruZ=1H_T!;E6pC6L zZqn)8&v1#ETq>g7cQN}cFfaJm|9pi1ge5xEv1QxJW6=xTsjhALI;ZB0#E@x3B$$4B zGRt6>biG7gcTqhe85X@8K@a&I`dCykVox^HmeJ!|5L@2!O~ z>*ynP59xWC5gUi-6Z^jhlcnD*1CrzSUpsAm6)jMD#(^lmL-3%SMTXNx30KpdNrk4} z@f41B(j1<$yc{|!>lkrV#?!2Km2OPiU#3aRsT+pYaR19Nnw}_2V z;-;m*rjz))<`Q-%7FKYr;TK@PiX(Q56Fi8MVhLOreqfDj`+ohn%n~Y`8GUvaCYJI$DFjoODfd;8tj|8}8_7B6P!L zY^&CI7BSf@()_W1_~6b^70*cCMrHv>1?mr>mx z&{was#Ic(xmGGVy`@6oL8#ygCpKmV+ z*ZO;pc@X3aPC@BZNQtt&{w@FOC_ut14V8)2-Y5{R7EK0_FzY#7nVOkd8x>k|nu-xJ zFtn5A(rYW(viNtnU5v^+$#f*%*`1&k|d=A2dikLBr`zCn&;)U#w6C?x(>Wj;%wirz8Qg(phyuI<{ zlTQ9zAbX{%Rwc93*8Z@@; z#4yvueox$=Uy`skvW13^@ffF1T>c(QBO763a8+~We*jwYV8<7QAZ#DQ0>1xKy z&&jN$@)pWuTc$!^v&$Q7rz(<21K<|D$Ahnp0_9|d%B%>E+0C4=;KcA`(u<<_jC2jE z);CUY%rGwh9YO}k?Ec!#WJY4wQW`DH*OQp#B0DO^?uCZYHpA#j=;kG$V_OHzR_a0{ogim*Wqpq2#Y;&%LWTsYqXlVQ;UD@|B zdcMqa2*HY))ipPtrh#6Y62CUhgrFJtIz|DuluktX-(Ukr*$M*i|A*uX9?Q7#oe(w|x6^4sK`V)Nx0fN^C zIs&bIE73&Rs-iS15a;3PiB_}IS01(3bT2=MB}uh|(hc`@({#p=ECi5)Em%4O&R-T$Fad%m*)1CY{zXcI`^BqrpHu9!KoI; zAiyRq8;34WZ?83C5!>B%F&x`0YEh@>OFifCsW7MPu~MOhfCR`tt4 zlMlJIjwzOaOV+PL)oxPfhqUG}lMO0}LaJk7Y z`d7U>tjof2kZwVq*|S_V9J^URkWqcs53*;!cNv>dI>WlYj_P?c=hM@?jC}3)!s!do zFKv|JKhIA-Q_ai!PJP+f=s= zLN4HWk5M7J_Iv%&`8uw;Gn!&UJA?HJ^Q2I^p0=(uEvuS(Sv8MyKzU?r>&CqHyX3SM zG!bo=j|ZAx>kbvPE^BV+L2n=uJ)YYjVtJrtbz3t9Xl`IrSefB?DygCusB6{A*mRuX z$q!z8XUF|+MQn|(EYVNfvRQul+WWT6XY=kFW4okw<@#o>L=*}8jBFIYE%X8-8PzGT z`65S(-Dgmk(m9^Y>^~|IV!5Yep72C7NS&b6UTO+Qt&;9XKd< zjI*sJ?K3(5$n6>BgsG&}O`C|1)d*#ty%GM*rf3t>( z)_OCcbW)rZ+|J_7mlU7W2%tJRANYhj#YeoeKDze5p1}eh?R_N3)M^ToW zq8!})K^Uga7PGH%-r+l%y~U)7yGE-$$;$dR*z+&%FR_O)wVv0arid=TZ$1^d#JWz; zF+R^ASL=~ac+QpQo7T`wFn05{C8cxQ;=#diB&mZVI_$9`=hqr8(o&#fgFG|EfrhHJe;J!ld z{T_q^*f}jRoA7(PGL?2mo&P<37}W?PqqLIY{>awNYV!91vQ@M7y#8+b>p>4+p=HYJ z^~C4%?lC`?9M5CFIK!qt#vL>X?0}y2?spyhxXbynQefO}Hhj5_pnp+P_zZ158i@{A z)0El()6UOZsJ;^Y8qz2&zrmS1UfyH}0R*$G-xTowAlYog`wPCmQ z2XwBnMkA(_OtTJoGGOs7&Qan9uPQAV7VEwKltSc@VkcX#NTs@2IOAEsh*{T!k6> z?q}#09Yh;EQ{4Fz20^wv-kWpUMh6DT(GN7l%@(_}MMB+;jQv%_US!H3KXJeW4ZdRi zxO1pBXjO?i`P0OQWaUYymo>eCv*th3(qpNzD%vd!u>(gP@~{P4wD|Sl3o7|YLl4PH zgE$Q1Kc-m1--USa>IJ~sf1}|_qtU|4!jJw!nSUl23HqxsE|V!au*34w7GVHBmtkBKmLBJBBK7DV!Fe$Q zJWC8q#LX8X(ziKW2SF`#SdITFGHqmi$h;(ba-sK|eIBU9hr3#xL&5IQXU018=`@r( z#u03qvK|&qK)_-_36Ch3}%{&bh^D z-$iL0w2t4DB&1e>bEzA!K~`atC+*ml-wKYibV6D-Jk}^`Z8z;16B5C7+G<#sJc19J z-un~b%U*SaBu#h#nPx%M603s{!aP2)3p?a4FTN@n(*~NESynV1m&#Rawts3gG0@4u zC{QJmGn#2MZJk+5U;lvUDv;Sp^$ZTLw@j(%|% zlQucq_OW3xkh-fS=f=eQF^DXu>$JoYbUj)SQ|>Rvc{2F&3r+iGh5D-d?Glt#{k!A! z#Bp%{g#KEw;mP}&GC}ut?Bk`rWd|Y+o%ihUs_A)b8O88;_kMpv#U@NFQsqjf)pF>& z^8RW^j($>$X!^YJy7HOfee0>|_zZr#?C|M=D+aOLw(&n_Ax9G5c(T1;D)c%+(NeU6 zIvRrp8)tYuw%pHlsyi-UL-5_ra=f0Rb_ghs#y^B3a51i!n3b?Ucd`{{Pg^kuga`fo8mpomR z6Zg=MzIc~NH6q_EkAteu0qey=+w+dkL4z*T|M($)a40Pi1r?RpLBeEmJyWe~mm|?h z)f3_x#BMkcccv0wCuATzEZ+Hp4vMAVoB@XMED6!ImG%Co$dbEX?+}}~J^)J}WFvPv z5#c{h7_dMehQC6nQiAuM=OCek+==GR$@9cEfc8Wzdrpe9PfB^D4kJ-TwO%Y1ca9Rz ziPFDL;H_5aMlq^p2ctXs#p~w6_^johq@?5RB;sy+H3QA)7Jlam#j5FykDW@=JMN^+ z>rdX!1%P5lmY9@BN{eAbm3|wH~4Mv+3(Fq zlXwNldKuk2gc*WRZ!jcGxJDwFNcui{N9h@gC$Ubx7_^qP-Y^LJ`*EJ)8HO|@xIXzq zHaIq+DHjOdLukD09iZ_Hm-AHK1z@WVim6`~bif}y8lwRSrKAtB>{~_(gvNV}p%M?6sNB51OVDqoVV0 zxK)U<{{iM1*s>89Cl>cUq7F5uI0zFI5PM{g}7~Lzn3)@osgocl{zU| z>15-d&{eCHuc?~~DW82Nt9}f7dq}Fp>?oOAchR)HHFxwl3}exIyts$FkCc{SD||}| zI`85M(x3VLOiKRDoVffpSP((info++HZFq#)f`J@n-g5j>HQnNj-m{?I)Etidbzqw-fjrkrzA9jxl07sq!$?it58O4>elMA~R zR_&#tw*&Ghk3`~u67siX>&*x9Mw%;skdZfeOaLe~K$(ryT>ywI*yi~Xo-`z@D2ks0 zaWqaIJ^Uj?83+angs>fbz@$vHY}jPf_*_f`<#yL^^Ni>4okdi z_K7M?W0v&PJZm(S+df|#XjC|+9fJ(-d{3J>-7iz*71*LNn2xj5@PpI{Jny5!3niqI zR9bqa%2wvPPkqv$TL!3R@<6$Jo7v$oR6PWg**tpUyUrb}6bz;ehO9kZI-WM3NT$gZ9m=-R$S0Zg=`EoO-~ZK zz(I?=Z+C7wNrN}?1DEXQ)&FbgWuYky+DkPXl%$c3{Wzx~S_?U8UPL?mr<~Tze40Q* z_rHqVMY+rxx)R?BWkQ6?lS5sNPo{u~tXO z>bz|%rtFfvP>gs5*4(;Itl@{*r669nhv@G4Q1DUQ=Th0WS{w=XlH=OLe`uvOEjj$X z6nNtAr?+{#7S+lzYIrw(W7$uQa+8(Nj;md}iZ&kukL?>7wO=Vanuhq+TT8Zlbb<9O zzdpUEwsL%Cbo5-GjC`KQ`aNuPUY~`!BG8qA2>(2kROvKl-=2x3t};2kTKF%20ZA7g1@Ee(=JpIi5>8-^s5`&Jq?K8>97naE0dll1v0}zMsSs zG3DmumCQd!9fB+(N_pq!@?%Y#t;-1$zXY5+)4yAE|2?ViR`DtjjV@fBfk<0-Sg{yc zXvZ3>Vyj#86GQ2F$>S~{q)xD1W3I69?2wtBMacpG=qX}B_ECY ze3{wcdwQXFB@acpDn|MHxVH0t5QMIVseW~fYMS*>^!YjHg7~tP$l>_!eHW5XHHQ;n zQ}Gb{>q-lR@?&3`-}!ue!vA{CTqH-U8;U^hIpV>maP<^(U<007R(~9l^Bq)D=v06I z#o%2i>_Vqoe;z`gvzUs<6(iM-5dCNnDe;m$Vs1zCym$vm2T3hx90Y{58VQP zTBK#SbWqAorQLm8eewI8(=tgI`o{C|Jw!Ep?SNjt0?l@ix7Pc<0RN<-czB@pYxN{x z6P2Jd5OLG^aUQ$$hAxlgs{yLoOc8k zc`Kz8ROSztll&(P2E|^TAxvUP0@UeaLW|g9lTB0XLhW#I4!mndB>E^6Xf@ilhdMGp z6n&IQ4)~Zt+`llymKT`^NW{pog4!SwGF_J$MfwF(Yvj06lg+?n?CoZ-;k3JVB)tE~ zONoo0arecY3boL-ELBJe%8Z8y{+zH4P8JE28%x^)H^szzCdiC(L_}j;@2*t8KuFa3C5e2fh0tg$99JT?biYn()4rw#74HDxw^4)&7K~}L7sBD6=Hd!cb z=TprMA{xB^(hes^xW)8|+ol*$LSP3e;4!62N!7^&UQ|HHaG!6WXp}Si63E~&FZ$1T z^r&h|oqkY;8-kpjFhv7@G2cz6zYUG{pZAmkUN90V!;oW zvmw|YI~x|la%TuWbsM-AUbF6UL+Eqvbp=j!B@d+6L9_91s@$JuR|gH*pA%Q%UbxNH z491?N!>PE<&iX1cz*)WNnsken;UL@cX6bPku~+N+<_^-2q(#B*_8up&xHoRAY7CF2 z%5daVbvD^N>JJt8*N?YQflrsyxkxOF(fiG!BX4qq)J<8MwBY)Ex`}%S6)Rb?RIa!Q z)r!uLv!_yY5byUN7P43*&fG>v{g#b$(lqdJ9I}-=Lnyfb;Ghxu)HbGLdF7+SmYZp2 z8++3kkQfq!!CF~jy8ZrQXmYIfd0KAkededml`MrTw^p#M%XQo zI67DL`c}oh0*um-yqOqGhDxQ!Fuls@emPu^N#rCu=tMmbrhFJ1I&G4ld`g`0BkI=k zC19+57HW3JbNZ4Q6pDea$co8UomomW+5&>ic^ z*?WwGTq}Ci`7cKk_oGxCegz}i_)$9;1^$7@Xb2qnjQh&cQtmrJ$$ps@c9F6U<(FI; zhaizE**{fzwQr34$A}|Vgv^xIg!als>}+7v@xYRN1EvW?o|EJ(t6&HXC68a2{4W?c zN)#@Wec|)TF--EtgeQ~2Ci40sqyjeQm7$|r1e!*%GB(5I)xSV1ur>u%OL7kS#2fK4 z#1N+@%sF>$7xv_a<@xE5U2a41)7#tFl_0dFG<5MDq!~)AH)7`bd{XiF_l@8C}oiYdo^eW<{hv(&61i(kz(%(U}KV1Y9`Y#2W zLV73n$ck@e5*>^RHrsbC$O}-;t3};}eCo`W)IL?ac@6alPUve=dLG}9BsD%YlQp-s zb?Nc1YvCj8oq2%z?@x8bs=00cYkK@Yz&3LTMJppjg~3Bgfj15#<|qZch&g1;OC?oH|!f ziZwzhd+fNth}gFVZqzWK{I#bR+-ib#A! zudVL3VZqDqT}*XI)suG`o}z;%1jr{mC`g=e0%5wx1K_=v)Oy?>iDRboL;vx9+{gIb zf^MZ;rRy?v;vm6mU+U9yU&6K<6n)F~G;H<(h=A@JJ~tB|k9^O|8z#fDVM5c3^VebI z+1#fffX1iK4$ULRb#oH6n%9*FAN%$00AAJu#Zp{TH6*_Gb7(>I%WMdrI|%-YxYG!_ zWlMqL&mh{w^Rd{Bl^1eY3~&soE4yD)dblf{-Unz|!}I9`a(L#v?t&a%T%g>w_Y>ES znAG*{JgL++=-Zj|Jk0j7hk%a8gO>94)bsIQM0yVt$-{D1`tgdc_je3?+ipGxWQ{sR zmU|TgwUmKy;XZXUypWrWjwVD{>YJ7yW<@1@Q_6-Pg9maTdzmE&(6q)Ny7O`Q^vHFc zD~2awG`J04g)ruZ3ezUGRI+ZQjL9)>)o&4W_l+>hl#iLmpWtYuHN}hGuktr+gM9rR zeg$-%Gy^?PSXQH|Zqd1EKPFV9A-_zk3vEUG|7iib@kbM|lM?-}QX0QTE$U&t!9jTjh{r<6YCB=)>Yx%HV5!5h*PRk>nj=h$Zy_RakiX z-%^bt*@NP6jhhGjU*T~rLJ<2r3;v@bMTDnjbtu#Y_zT%HRCj{d+tNskA&k(T-Jw5W zHH`b|5yv`F7}3y?b?`2Yb-K0Kq*D;Ozn#~)l1flpv<*SORSo{-*W*t5dyaXl9#8g(FwpGeYb4FI zdxpkmcZg$|Xz`ZaU}6QlHhUV$vv!LfNHsf*dMdww#PfV^Kv=907VR6T#Q;-FEp_Y^ zN)~0mgC3tzDCd>mB)_3Jc8p&zGh1V$4OU8HNu-d8NSdYBNZ(Wg`Hocx$5S+O2HQVV zZ&Ik>_Jg*MH;*pm;RP#e#1|L$i0u!(gM%4`~ElO~4tSe;;%uw@_SZS*=K=;ViZm@6l2%O3B-vODjM! z>k<9}p{cMUIg<$?AM4A)!Z6n~YGXmIfu#epq;n5gm-ou`651m?5TkeYSA@a_=sXE4 zWdmoKJ#;2GWCa{c+UO49Ehh&4XDp>074Y^|@Lr)B<`MmorB++dO{_E%HK}ZRMqo&7 zeZZe3+HOPY+wI4xPghmh4Q#a^n>(9E0RiS-bDj@{9eph8p%0V?9^f_449XvBpX^qz zIoh2nDvz~*Ydr~R1LNk}_RK^P;!#J8* z{LpLh6VT$qK+Wt@{zst(XFoEY;Yd85@|qeS0ZX=@O;rjcJx2(4e$CdD>I!e5o1u)3 zt%bnQ6$1D5EC?f`{8?1?H>M1$L|Vt?Zi>~%YPnL(l&2ZwS_CI*;{30`1hi~Q-V{v82<$)ZxuJS8LIXh%l_UsRY78w8)8*Fo)N$a-S?Ipp; z5NF({83tAc{QX8;iZ_LYL+KGudhnheWz0}*%mDXRlE%yXWyvZCxi9pa-pI6y@)WiP zBE7MXT6%5$Ub+kZ1jBEL*+L|sR%0Rm@c0rc>wghj#`37!lhpZT1-L)!?(kHB@x(D~ zBtRxJDYvWtnw{{4b>`kj^;S?f&REn>N>2HUI)jeS%ADi0n#P1nW!>2~JcN|Jo41!t z@8rFaU?xo=G`%fME>?0ksXX13VmBtAEnJ7a46r>(_c`rb=$$p?+X!}|MHn@y znh3Gh>yG}}1r}AZxBT5u8`;GULdHpNGL`1=3A@dC5SB`VKyF|=Y1O>*dOb$Vr8nI~YW~;S*QmL~D+u2ZMZVW7#C6Z-uOR5@6 z(Glw^rKxd=2>(GVf+kki{JbC%JhL2$#q#9oZxEa$Ke`E4;IuQ<6*6QoDMYLRCUGV$ zHIy8wmKIiw!>ImFG0RIO^4P8+EVJYstQ8$pX2u*@$cqpc{pQM3;?gNh6rD&iMSif$ z&Tz-2rumRaoTYK7KC}Pnf1BoS4ofKoLS{{s5wmSTT?^dExkzG+ zq<(7sVj}_*?%eg`V~n%9H)PY_O34r>Cspq*pt{Ck8cW3hu7F^=Kdty;ck=az<=S2y;y+I}cC~zxDqbCRF17aCUcvON#;0P?uFkHm9DROQzEkU- zmN>bb1%IAqm;W|=6YCuhpXj+!QC0rrsav+O@##^iz*jDE?w~B1FHcz94m)qJKf;a{ z@P^04_&t$>4u<)(KuTi7!8E$ZSs7If;L4Xb@sBexJ}_>s2a?4P!8?v7$vF9sWf|o! zyDZM=)#ips*VW9w?%RV2bZWmiS$|98(6*PvmY;j}{>_+_-pZ<4A7;Ekw~}StA#Q)r zS(OTQf8W^6VEo={KLqy+jYhk3^{N10m1ark? z3@w{qK(ljm!1s#zDoOKIL+~huSPwsGQCRgx{)(7J&gA6eMhXKFRp;b^ZB2hvN=xXp zz}nx8iOs(aedV=&=Bc>;dcx$+x9x3Zz68v%QEu|=YgCe6;#s<46Ns*1n45jqOMXsg z?k4Gq8etwzA?+sfqw(uPyd_B|3o^Nh5rou;Cyt5tG`{F z3^_@T7r7y>KiQUR3`3kh5{NRkoztmS_KTewO7C6CQX>|s}CSo55#rXZ)CR9>% z0yuDz7Cjv;_%ckfMuh=l_9iFqljN7versUKsm22JAw9~nenrZ%LCX4zt)c(zoO2i@ zAKDbi^a{R(CSnnp+gy`ax*=o=gRtRe*54iWNl*J((YeL%Dcq(3>hG4X$`IgTT7fGwi(p`jPg6qFxZQNR zy#73Yh$MraGCsq)mo8V^$d46}$9I~w-s+TI7u4;RPEHWMMWOmyHhzd>@&I*kwZvTsHgaVCO6BsoZekxVL!GTkCsdhN zM^~^P%ZHrE=cl#uG*;94KfGeF?F=smk%=HQUKBT%rs4P}@p>P*{wdAlt-Z{#+ULK? zKc@}bu)LmcS%KGHw$H=uZ3=hDdX-`L-;z0>9Jom>7{Zt<7!dN^6rfMe|5iGfz*4_M znEM+t$R{Em3$sTW`>AGM-$^I?d&*5|yUMhNMY~cHjUzgcE5GRfgk6wyOnz2m3*yG4q1Y4eFodve^ty- z_^_)w2^Gk20`npwHiMj;J!1l(I5kSpS5`V}nB8jUTdh`WU}KJQj@jwhV&pGL7WWm6 zEEm-}ovIiXDV|j)LsIPgs*}q9ShO9Vd9Bbn8JBQem8JVU`Qy%~JTS{lo9Mt^cwtJZ9DuqTZ9uih~fap+U{hRJjF4^yRBCL4nt#NZ+e<FsKNobPZ|-ilW8;8sgSr+T>ZV7f<*sRIr_XSdiF&5PT85tHL(A; zhJJs(#cQ6mk7N>8LwM+lx**y|A}Qc+MPED5WtJOQ+Egx8KdRHRt<3${oJ$If83*| zt^PYFc_P4Vh|tA)ZF?_RUY2_u%K!JIpr1RU1_&*O1w44h41ABPLY4y z%S7b3#UZjy?3&^J^l55qQB)FlpdMO%2NPl1NceBmtIn%`)Yhpm0vK18lE6nWl4WAq zC+E~ByKna<-?A-v{mv#;lf==#PXc4dzdi7#4TiiKGQF08U^);j6dc=qHolQLpvA0 zFdrRWdL9;rF~H0-?A2i3)x6{{Q!uK#Gq%^Ioe(7*U4+tP31y^p89YEqi>%;EEZpZ#V zD|WP$M~(?+F_gM-n65F*6ivh_JC)V3TV!er5X+*N&PX@Aa43^xa?DhmA2++O)@E~RXc?!PYg0|(@K_pJ(l86sw+qnea5iPwnHGOx7p?!M z&af!7&y&w3iWWG{x}G?b{?nIO%-6&&NWZxJOU#9DAL(b!52ljDgz;cL`saUdeJ9x& zrXqEokfuOH#=E5D=UC>st4WcEenb~E-FOX3tV5&ciUz*(v)zajoOqik0$lkT?(&Al zi^jy?i8`JC(*peN(TES^D;<^{(tKdr3i}?gL<%B9+=Wdwm-j+#)V%AfAzEPjLM;mt z8__6BP^~&f;3@qhqL9PJi^){ zf%8P>!ey7zQn+ucrTRo2dwb1Dt2#CxMlM+TZ+V+}*B%&EIytwBXJ^voKUS4@%oNe) zky>Agk|gPR42ovFq{2VO6Q8-1&mT@pNFQHDw1yrx`qQ>r${TefC^vq0HwE?Ez}U-Y zW1Sa|ej2VlnT(2IPUIXh*dOGeL*L~3WTWDSy%P|y5n%w_x-^Ls90osZ{;Zr|V7D;Y z0%LoI8C3*kt58*CD-xxq1&agE=PCNa${S{qw6j02Q4b`O6DL>|!__YPSCMtAod|{e zGbepLS{ThMxnu5bkNSne6{Y({Q2%XW>G=I*gt2o1F+0Z`5j`{JO@%&c+xhKRFE2;x39Rr?uk3sw$@Q4JR1Z9m=&B(ef`y)PIaB z;v1k!Na>qHKODCVWT9MMv-j}OWaffhoG%SQM z1mo~rgbDr1QTCEEO?G@W9RBLU5G;zkBLyaiB>nbO3JFm4!m|P8$b=*itNAnvd1xlJ zmH(1$?#JSenQyWQcSU^BoBDn^UAszfjqQmHIt=vcyN5|cJg5%#b zK3Qkcgd#<}@k>#KnqsVWy4_~LNZ7sx_Te7Qp*(M43o95& z2BfIzLoD(*BlUnZLAH}S-JKkvkWOBYJbW(}EhM}K42guh^Feq5BS-ulnI+62Z9;Bx zY}E1I;dzK5GE?kdcCGSyvy^Wl9DI{+?D1Qh-vJ6KOr|<1yGbrTs9EGATI4dT?2f;& zT|wXrE{E8?;vzbrp&nK9w!nT;rQ;%xXA9A$4nX>d)Hd@QB{jl{FmD11{DrY;!<#kX+TDWV5g^rb(4*#+&f$5_P_Mwunsy0lSY5&HuQ+s=SSgILIESuM6 zFU`!k0nN}KWSAhyPt<>#XH15lS-P-+%{N<(MnBRNXyTHqnqEIN3OiCzS>V}>L48EV zX`t3SBNJIW8ElIR=pav5Ds@p}yeeakbE0M{py3J6=!^d;S`q*j*wtK}IA526k=xcs zrvOkOD4`)=_+Agtk<#WZm$MHj+&d7SN>&68uZu0cQd=0A3qbSi#uLx+&2}$7D)ard z?DVs%4`AtoXcvJnh<{~oNS;A4I(&Ksg$dfPX z8EV#SPhYc-AR$r2XGp>TKG0nhcCpmQN zf|6@}&RCGyD2=R9%&k;R8|7bQOgk=&hPs=QEJQg` zmrP(>Q1w$*KE56%K3Tqr&Jvo;7`mP-0Pb%%SDA|v&a&(JuWOkuVC-;;#WhPDjupZ+ zY5Zt=v6z_!KxS z;xBi>Xo$4r*Dw#|3jgz!w=2hoDKYdv3-UTr7~3f%9s$9IM}D-3;pKC8!tsCdAl=?X zH(E8|`V%Lg%Na8U-BRm~5E#9z%faAn}^i!$q{zfLH2i?c<*O%h%vWSd0^A4l_`4sqnKh=48HbS=?Mj?~+f6l<2`ROPB{^)^Ua zUnlUYEY-DIU|+H{E?3vm92WjZytir=5|_5*SQ8mVB+ux)1c%qWC7Jgp>Ak%IJ*FH8 zjM1eS6<~^dOW?Lt&H7)VC8gKQH`n|ki=}D-ITLwtLedcJANIzmxi}&Y8d32V*i?DY z*V3!p`nNaRU0(CZfCJ&j6>YjhuFV~bSL7VcufyP6Xm_&@C$3I8`mGS$e@Se_JsW)h z9h?BjQfH%Ua|qktcqt+;2M{Rl;;#K`c!#n#|v7$8h(#l{Sq;Nt_Vaz$iuIk3&w8 zNa0J+2SL5M>byV<*25^=>Uz;l)uz?6M&#A zjrHkhGxW06lcw~7_CicpM!-68d`kBK-Y&+(QyKY;G4a7DuHqj^Wz<=0HY!!(hv21x z)T%M_eEB{uR4PQw!_aaCUQ1Vn7LF?bAaptg{22bNfIy0>3K-4lam@I&Bt`l-p&&Wu zGY$L_r1+|`VO-jvBHeqV047w)#NB4080`pA?#M~JugYyrXWVrJnc*yYQfoAd2yUOj_iLyf6FCO7{|&+=4Hti7aD)@aE8xyR=M9lJbXkr?TOLYONe za~88MRtJjiF68-z=-v=zry%opJ{mGFMBU7Oscda{s<-dB2SZqtKJ5wreXgA(6d8-< zLqR1EzmfY=%DG`Vgu$l{*!LU!<m1CAtX;@#J zOv&|B+*|^_PJA0B(`JZo827f;{B5cYpl9ENW@4WFveuDx4vUB{5FlMZf!Q>flWZnV z7yhI-&KQfGEH0Eb%Q-w>aD%nwSPo&2ApsC^AgJ($FL_W+8xs2%TG~6ln2oGtM;Yae zklk+VcG!iePoS9S%12CJ^y%f<67nUc*=ZVO7E$t+d=7e`_YJM#&Lsu;RTB^Xvk|CjAk=m9V6ZRviOR+g`GWOoag zb6LJFh9U_&MND}ztPVMG$-4Sci#Msae@$42o#`Bzvf`yd69NAntOLEK!%Sd?&z<*l zv&b}vEE2!RYKZE}W%Z31yRAyk2;T{magd=tRi6GG@G=at24z7zWNc(mkk14U)}MtO zPX(9^QS)%LVxS*f$nb_EL%R{6lphXKVmK#tps^^EEEGmF$W`FD2OlsQf;A17%EhCR z+A6e+SdSo*(1IVzMlu+ndN&8?|DA2p*i@(xZJo$~h;%YU0p|ku1ParuB*-AH3oa4X z3q_GVSCIH$aM3R&FO`6SLk6bHm%um4rcYcrjqWGe{Bgm7Ij=6UP-F77M5$8hL8E8& z9ek61DIu3?dUU;J_-5`H zhZkVUDNEWOy9C2u>@y!GVtpQJD;Bh#m=n^h@Ey0ANNmkexrJ<;t@d>XWWAOiz5UW0uKP|uo z4tFtQa`)WGw^g{OIY0>pbc8B<_lF+})8c{0*@k^2rBs+YS|VQPhf@yVW&qNUu;p=fQi>LW#^r(VbQi0#@?*fWq5{_4q7#QEr6I%l!4GasWB*9=>g zxh-SxoaA&=D;hBIjc4#T5B6YL4(Pi=!Y(!m8Y9DyFC`{!X=11sUZjv&!$bm^S;+~? z4{l^xUU*JJg~A7`4R1|}1Rp+=%a`INDJB}M_BJsyy@-jozk=>u07BpFY2F%6RnDl^ zVtEI2dA{9fjiGR&gs?ofrXI9Wfz(S<&H|pEO%1k$0Ci^Hm}re8Z`n1jf1*$n ze+iTQ8@|C|jW7K2r%|V%Hh*NN^nX6#_Cip(c<5b&;G=BFY+Nc10z+0LJxBC2jsqVA zU)rcgKt^NtRt@xe16TF?>J>xKEkm_ACTL{5bCm4N@L$6^N&N}rT1&ERqs`f8Qyv{6 zkUT`vt>z>J2t#bkKp!;;>4fc;R6om>NC)n-ml#U}NXh*jNq9^0^$B~008edjLu<0G z0|e2{M98tH5uP*uFl0^i(BI;tB*Z|nv}2m@#3hvD^f^tcKChv%e_K+5QmO*zh@~MA zQYAhT04!>Uar;q#{t^$?H>tufB1%Higu`ivTL{wz$!iQ*GSXi~6!BxgYgVR62ay~h zO&_EQ-V%RpWfxx$wxL&K5H0kLD-jj(bY64P@3Zswht@o4@|JZgBj3!PB`eA<%D@I;LX@9uxx z(qz*n06~qrO(#<$-Qr`VdB)p5?V6O08)sVUZI9UvGg=n$A+z%0V#;PBV_Q|Wo8a)S zEk4Wlo;<2c(~r2io(VRo#SDtF44Hfag%$}o-`sG&e=0~~J#~puceK|E`Zzphe2ls2 zui{?i8Q1=K=&F|Oi^-ZhdG&^VAXQ2pxyMT4E@

0tlpT+JHdmPc|cwx2XLvCftwlEArF#woy}rbi)4N9_YTAVP96^1F4$$d;EE z^LW=b7D)d_qI)xzR>L6tX@3`WrM&*%x0+Ijv)eZUQ#jB3sbt(%O^wb1+v^gWBkVZm z@S?N*kjkc6{0)FP&Uronk6)_bB0heVn;fljQd^xz$!95&%0RW97R0<;?`cYVoh>=i zm3%dLv>1vf-UVle|6}2}HncRL4s1OsHG_+-wlVr6m7-_c1+KjUE7~_NHyF23)5)y5 zB*~u9XuW3jm|td~YWDRnRj-bKlx zW9EzdE9{O>$2}Su;kX2~zTlk6ejdMdhLSjq0stX9oMl%xA@eDz32zSH&GW5OMUmy0 znpA)2&kb*Q!j`UOEG{|i0LNdK>7j}9J904UV2X|3{7a}l&NWsAz~Lsf^WI+?7wt?z z(?5M^J6&i7PLMuy1qbs`KO1F7xo%(;*dGqCA;>p}-e4fMs4-z#;O%bR^Q{;hHifT> zOP>@Jmoc>Tk)^E?Y{XR7t})` z5tJU1_>%)Nrp)sl^}9q2n@Lp&ahJXFp5(7x=6B>~+ip+{2T97$ zwzU{YRjy(ekq5cD`e&CTn9093hF>wA3}E=>l3__ zG}AC1jQE8JJQ&K(-9)o*j^=SGnNTJYa|(#Xoyrwuqj_zVZ+<^FxIO?Gd!utP*91uQ0Ra2 z|M(){Hh%}FpYjiys9agUiM7j82e-Ze>BZgpp(LhffhJ6tux!Q3kaxPMUy%VV1|w&P z!CYcFD#?0J&fadak7_xs4W9d8cOHan7Ugs9Fq7%9AB4^vdW1{n4Zf4S{#ilE$v$T$GZ)iD#wh;S(k-@tuapIJd zz}+;pfrhxZRVZpB+GM~wMCHc#4lS|)O_1wrl5%KWJjOffyijMPTP=b{p(#HYfOCo3 z#eUtN18UTCgx2(^Y`RV@A z(I)johqu>0&&jG*MW>bbH<7HR1#{+)~2t^If!5R z^{aK9c~7NDZ=#jP4O_h<8e6Q>kpY@Rqn@sD6LDSXdc$MrL+~-QH1H>>*-fSc;_`)$ z!t$HFr-B0leVQ!=XXOoj7d5atlN^X`Z_@oa08>a)M;;^Rh=)+0=6YH=G9ZJXwgGs6 za_)-zV2FAkQ%U^l$0S4P|w`$7 zDErfG@P>iQU7|M(O1TQ7a7K^A`E+P|*%p)&aaYTsqrpd8Ih;}fZoHAHTXtca!M zn6{QvZ!HZhB(@5ILYvxyJNFdfp0a@~?xsEN8f(&6n@9$m1bhkcHBImhXqq8FR-JF9 zp zbn}}f?Vt%BJfcnW05h)XFI`V8VWJgmdQ{)haAIjR9E2Af4a<;NS*R_ znHXs~%G^TS_qixR7rJZ&$|v&g7H}lnNEBid#W_YuJgZ>IA!vb2B2#5f#;We?6Wd!pNbQdujM>oDt#nOR#FWB z?rb$OIRBWQGqRTnyb5?K1BQ$;h1?$eNsz%xZ8jMB6Q2)5j;cGrj&!o)owaP|cOj2u zZ?`7N)48$sEjSyZl(kaB_(7)4|tVi+^0Jra9G zr6UGIPsxXdTE>5vdeGPT8P82h{cTg;_cD4~K|b zv4JAeT-%N-sO{&DEvLpp2He4+e$cb2VJ6!<1c=+h6xM)pgm_w=I;~RoHiD!rtuvSlmPA0O>B=~hx%l@K3ziRK3uh{1GCB~YNdoedAy_@cBxBU z>QV#JAsqb45LFP|lPbSmT{-tDAJyl+MI1@;b{;-1-{|w*(m$sj^9ws2^iPvcuTi&y z2L@e>E~aC(I;^*|@^isf6$#l+*yph8>Edh?4VKHf3G%o*2Q>IHHFr4TUO07iR&R$z zC`F`FIlbNW-0DH2_f)bzjd5YwYp;A)>sro_A$9CdI`#&GHH!vSDMyU*Q0p)l8YI{P zcxYp65cNHf3s;S<=gNu0VMfEI9D61)w9-z0^-^rriT2ighl=Q4_Wv^6;Qv%i_|$G# zaCMdKMV1%Y5%~`)(f^3M{GIYb^&C9qe>c8RrwyYXBtPMQ_ihRi>f5CpapRvZ4Fy#; zZj7A0gzFSe3AGr=aH>ejl#pSknq1l79AC8_xg?_+pahK5SdcN|LfV$<7AP?#1sVgq)JSOn7d{ zHPXuE)9{Xh&+RN{P#t8F=c=5}2Ic7dmph3T>{DATM;5CrGnh;C27~QTt7qQ2svhts z8665HWl@x}B1&0K>I_KQLZnrg=2jt3Hd%WB-M|JIZGe(Cpl~N6B4w<*pnm`W5CBO; zK~xHBl2}s>)*fZ2Pu3$Tt`?*zdIO1wB${=|niR66XG#Lu6q=MmlNFkjLX#z$IRpt1 zR5I%kE#5~_2$G=o2nMf0VuX@FO(2*Yp@!mjQ*3+zO%m7vUhMF~GoBQa}wlp!5!NavyyoKcX`-#W);WfQK!dnz2> zQ%Q0tAF>@v$e1aauvTcoTZZ8*x#ZG+3LfX`z806^oBWw>hIxdOjqzPbKEo-Qd{2eF z=MUv4VuX(!A`c1aEQhEk-cw=s3<6-1G=7$Ik_ze7(;>$cf{3Q{;Op4g{m+%)2QjeorN2N)%)}Ksi`W z0NI2qI+Fpea~{ZoT%j^s^CHe7#d|6YAhAz$TwNsUlBo=#Y`}YjbQb4>VxFk7Q01w@ zht5+8GdIG_U5)bH6!28mVvV<3;4r(D>@3{r9LXHb0o%!xyH5I3BC2^`#J9TXbyIu2frd?GO3jF| zCo{cWiw;3N(kOE*D+`G&Dq(eXZ>Yo-36pZapvz9&VDyvny8JX%ew}VM`c#e)E=!T& ziXl#3cVgvCj3_@U^;w+BK6^F2D5%r!WO~t3^@fUpKFG-#V(9KzR$P{9y(bT{n4HYw z>_#lRY(vT+`{|&n8vXr(qL+Q-MtKN~Fm9u0DO1SvoKn@T2Ig|8RBiLxLegGc-Ah&s zQJjgSOw1h^LR48>Ym9EsSHOu=Lfot-DJS{?M#lgxBovyp_Si7p0~sT&>7Y(1#v1Ii zBdnwLs=FHd7Fl$yf9MEVSccbQzkQQZtxH%MR@H7q%MS*NAXgcuM6kW_f?U(1vgvJZ z5ADkK&p_~M@NlMsD6a!n)flfXb*W2TYD}l?-rnCHhg;^)!I_!+}uRvw3S?AEeW}<6Eh<&T|raIfTU9s zxP>}vK>NC|px9+}YJSFIlla*ZWN^(MM4 zb(0*q12Ba&)erRnblec?)+y!PQ=Ob_qln|s#`sFcbn@48a;{8Odq&$%Tb<}JkXJ8d zEF|6b(8iv&1&c(BLwkbDD5+JW(Kj&Jw?N9?VK>^jdRHX8W#VqCK5?Hrk+MIbkqnn@ zN$krAs=G`5;X0uXwk@)Ova-Qto*h%7xm_AKafd%IZo;T&Y&<6NCoG&yb*KKTE9u>A z;3Q)!igBkfGP2F5QFhseJ3&UVY~))7wofXVW7)q|;QB<%KHji;2Sv&h zWC~#TZXoBW5}cu3Y_H&%EDGMUfZvq2d&_vgQe zA~|#wc394ODSmgMjWuektFbf#XU9z94F>TxL&}Qs)>2krvKB%q3sF`ft=-#zwg;z` zA!!f62Dpt4P>Ng1;2vQO_SpdI489g+!jq3lWF^Y=$dajfH-%DY3S@&ODKwLVCUjYZ zX8PR}szHz=1li;02D}E1B!Xc8M+lk-K~k<=pyn+qMoq*K5| z7B9*_Eb($ikitu1G$6dFOI_+xmztAzzQHMDb`kQ8QVvm!@(~@P2I)8we3!wjhT%Px zME(pp5<%`wF6R33(I09*DSc_g!@DDyoIjc)gdD)d7tw`MCVhUK>{_Z(QJaW-t{CMT zQCb?}PU_the1H@uEh}i_C>Fvbgzcyn18Tv(5$0fW`hv-Y@!2dvge4Je2`tCUVg@SJ z7dfGOH-+3B$JNDh%*dRaMa$L#oP*3viT6}YYCc6t zPS!HRm}cLj0kA}BB8D!JOl4F99EVyodN6-YDk{UY zwTN6aG1JL+fN}4pxbd##aEg&oW}o+hZ!yt;GvEOXzMEn)GdPm(sW?9)<&c*e?|E|d zlJhx%+qyEc=OkoG^U4sw+AX)&VR$nPZ>M4y1@#>VMxT=joGj(cH%%szlkQLG&rN0L z6-66;v~{g8nJZGJ2G^;<{TW$nNT#EfCS*#@j*af;7zIrn$r0jhEhJ%4qP^|kP_gwZ zb22CzxXn3J(rpX38A^OC%aNa()a_M$>Nif`!RZ^qnHeI#kD_mrMVdOmo@IItsnVqv z9^rSWG$00pWXO|}ydsl`Qf(itFS5Ostj=ykd@$PAw1q=F@V?l{NWU5=)858P)%9>& z#0RN*P=#UtiE*Ah7<8KG+_BY#j|X5;ZTmr6%fP!i)ST5y+JG?9Q17#VJ768*uGLAm zv7K&>%t#~KruVW2l0tJJotLhyA)2cJK_~&*rJL9u!4CE5u)z)SO6!w!I?)5o6zxoX zkOzZNm%7xY7dIuMkD;L<@L{?Z_^??iDLIp|i@$`Et+oNr%?Pa0Rs5m34fXk=GyZ|SVw4vUC78#|z#f?40vz=|mA-_n39-_^RN{TOvf&CH#e^u&-*qpDPl zTb=r>ZmM@}qP-m%&5-b1?I72;;<-f`J3%YTFGBTDGb$bzc^8N7!0m^`+bVFIeP6_0 zXSjp$2Ky7JzxC7KU;QgcJ_4GtE3eWIZ-CFMi?VI{wp0Tm%6kXD4f{!G9_f^ zH^mw7>>B{$oH+8t-^z?-V(?|+EV<&OWjI&PA)gj98Gp(n$b>RgioiKLlCtfG@}}z6 zaLPvP!mW2W$wKjsmX@?Gl6kWcfjD7B zl4q>N_#`@EE#bF}g@jX+!X8Myn-XjQPi$FBCVZQLQ}3qaai`|Edz7PhQ#96!he?mb zmo!fyTR{RSM`u?P$dWtNh{!fTlNFkjMiVVfoprTioWSKso*mIf@k$Am>NG$QPNj*_ zN&+>3Ag2&Cg1fp5;APrjCv*td3Cq#4$4g@u`~x^SLSv5{UR%7#JHG~av6Q{n?NXPz z)TK_yWMUA72vy3E$D;35aUHqJ1Xyt*b)uYTHtvw^FnfNyy5i1~6G6rmIEmO4^2w_4GOmT76|z%Gy{7{C0FmOh zqglX-U7rYzHZvhPYfI->msO&phi31VQ8VnN>36R0H){c%8qeoOYvONW~yNrgKV=|TbI!_`Zq;Z!C>%mBJ$eDF6?bZa)w{YZJm<^?Ng2`E$_b}d)!Ou8gF{6&;FoT0yoX;`vj!0D_ zq0b56umH+9s|#Kzmeso{MB)vV*e?;zMv1}|qICXDpbvpQARFRXmLtL`9mzzJ(K8R=y^mWk6H>@ zKpbpCTZ@V0l+~FDlQy6%nfQ{hDAhc9Kf3RHdGe{$4bfGX9H`6Kfy9YoSpOn$f~Pc0 zVmLfs==rB{m!4BpzM--b42dII@rqKi^`zXWm}W)l;zN;m79&2R!R~&1d%)n>Z(MDv zEmmeox?Nni4`<||plo}YLiQHYjL?b(3UWJ3uul*Tq(zM9wrsQ}8{eD~TJ8v`COUU) z;Gq*^>jJqGqjvd6kki0))Xkw`Sv_Ne#U^I$A=$D{2-?ejy_h4cGfnRd-`tJ04-GZ4 z>~pfVP0CQHGty0Tsz%T#H01|FfgrRI#la((?}o(fvhm6O83NH36Q&nYY|5A7WX#XQZUmNZqW$}*+M47pj_)Qlk8sb* zxP~D;)18U(QyR7Jh{WxvknI09wImcw6_9?kty;|}TW_k1-T9=dE|BvbmYvr)i()U^ zRYFyMvNzjXR6J)<@1`_(Po);QOvg1e{my78>S|hVu_&|J`?7SeGXdt3;dVpp}?jX&gJ$2woDC@XalWv0{Y*+4j?D9+_ zC%78qj8-|#|2>pJIi8{CEGOUIo^}V@``U@B zmgP^s75b_A@sBA5bR*m^WfKgb(A2s%K)^N44l3 zl$=ZyDt6sNq&TNJTW+HFQ7F8L!s4wIJXxca3TJazo@6JKtzMWdNg4c2FdWcUXpZN$ z{242vaJDO!33L`3gp+Hm-&a66X{DiA(lV14Xutx&y6WCwNLi7vC`0kv3<=9gb+gG? zh(8xEtwLLf)6kH#jT(C}RQqm(CZ#ghUSVImZ0A4AjlDd>`&x*UPo;aB!dVl?0B|Ovg^+2KoXVex;B1GF;}k3;$;;Bb zkN zDe<0)%z`{8AycqV!Cfx~&b~(>=Z{lN(!;E-0hY_U45An26fm3CG7(%FngIX+5CBO; zK~%go;_LnY?7aniRmZkJz9t?l1eXLtLIihrcM3&Hsr$Y6UB6HMzo)+^zxUpKSKig$ zy3$e#lnPX6u~OXKCAb6$2@oR8e{GqWHM3`*oKR?cZ!@2C^~;>S$JUxPOZGlT8>TPj zOs71cTXg%=*E4=EHV=r{wB};?x&?%XWn%T73Q)|YJuLeI`ksnuOG~c}hzAEj&(3&H zMd%qFD|?km&136Vuui;{B6b!8-&0}1_f$Z`h7J@I0yQc&%QW#JLL`Pr#ul6aw@k)R z)?wJXRl;a^VG*(QaOH;5wAE{v)0(gt!tJULx^{LP&5Vq>2UjUl;Q<`rRHa@cAGrIfEqg{z5PWEBC7cs!y&ey zWeFlqIGP(OAGp@A@;V3>Ci<5WkSNAnsl-;+W|br@;xMi9bBoIyah#RQM#w_nCK>P> ztBmgf88C+valWBri%O)_2i&)3I7F>O%+9Plt=i(5`@NF!MA+$;CAj5c%q}p+XgL`5 zcq(Cly`4(=Wgm}~U(qgGq)V&9uRC{wRF~>fUA}$Av@wVpx{d8CMHXmZ?<-7|hU;o- z@scMSShxXR*{Q8l?VP1FI5;lis#RIT2>suN%d2#06??4mP;OseRc_LP&3Lr`4?l(; zl&IT9HT#wc$>AY^sasf0(%T)`-61oJu(fV`U$))%CHlRkJs7w(sOrL67xfmc60Ot) z?P?qKN@Sg;W3yDQS}I+_JE}3FDU~dub>Y5_aaNbFxhf(auB{2eWT@nQ6hY;i_!6@q1|4$)i{?FSqd+lvri@_v44T5KD+>m1P}(zkLF_*r z<6UZhA4SK$Hj$ZawZIgE{w#(!M|UB)n6HM-_fZ&mCWT<|2djT8g0Ka(XM8-B&d7v? zl4qd3i(tN&K*D_#gZ?~}7(WYO`V5!Cpg;Kq3j2!(^w~5}PyK+1KN5+xyqls+DTGR9 zKmy4#hE&p6n51ckGHB0%8WP29Qlv``lM(^}X)&n&M5I+JK$D74dEinWLS+De^1U({ zfav%xT{g;Vg$-)wwdGw0o6hlM-h96Sg72n)Ml~+mI15&t0Mf5k++-Rz1Dw^LHS_nr z!A(MSZgTRBFo5=Mir2BtwbwbQaZ~b9@eXti<8v)q4(eR6pI3=rVkFh2x>T3y^5u%nMg?VofYO2f^#+#h z7;REY-+iR#j&ALUBUjVIM3s{FNPMf$Z(l_r#k1rb^fy!#6}psKJ3wuI=_Am-bU%Wb z@2n_0LA0;T4Rt&+qVvV`<384`M|r4_vqL{qwDV3KX3A<6?E`6_+9}+6l<)y_+I^2fs<#K2tq{cq;`&`;>}P*r%d>3#RrzXoi*cF1xTTeX7n@-G|~* zCdGRyU}E`X5~KM%7D`Q97ZesYmG*9mz#0|aqQar?sraI4T1TR};Cm`UatJ0&GNn{7 zjN46_fVpE##usael=(~EQ?dCPzU>O8*7ApA_C+m!`KE~I(RepS^}>ksQV)np>9yEt zLasz#riUf^^<~V+rm(J`1xwW%uKIlbEX%iJmeqw6=M9yTG7&lwwzb0Ix*(1~y%G&pO{!-oY*<+a z;tYo1TLRH{t8nNXxiZ-0c?`D=n^(Bw6JG5?13Gup8 zSik6&2c!*-%Y!z^cY-TjhA5dAxWq2hYNnNPHd@_s+c>NGtQt2>wG@GyRVjpnWR4Un z4n|z72pkMw2FvC4c~xDP`PopgOy(B&C2^$Thw5r?JThu_Yn-Ys)up;r7i%%>Sg%ke zbxVyZO+}O%BM~lB*IDk2fMsq#VrJ+`7R>8|jw2a>!E>LPR-dvH%;Vj^Cem^SXBJ^Z zRlYprhLEvXOH@5pd018KOE)8I?W>bIO3BgeTbT&}nLqP0E8*=|N`o`!rcJB+JKi-E zN0LknHfL#?eGzwebS#3fzUb_OTbkYWzV!A@95qX#oKcvhg}M05Zi?yy4AEU+N?j$A zM(t`Xvm#nCaTGeujw48QV1rsVYg8A7^#HeAqE`n|j*~9v9U60WxjiL>BOxAZCETyn zTc_A9xBiBTQJD5h2lxND`HqULGCzpavRR_JE3_jqw20%cLJ4W_FRbv0f7RvBtMKUl zh<(O`7$jE7U}DZ$v40cyo_FApu%JvmYpi1+&ygt%aj|h+dp8Bhvmq{WX_KbuZQFK? zao&;xTF))d&>2{s#N>H%(2W`s3&rl&c-X1V%Fb(*neELeGF8u#@Zj_50;?yjKzm-s z9Hx4L55&_q&6=mTZPzh2HWom8h8-+crh$oINl5@uPhBxPwmr9{p66ovI|{*t89>BNmtF{9yOgG+rnYIzr>^@roFYGV@_78a_N#C`?yKd zw6^WL#Kpz?BQz&Z4s^z1Vq?Okh1sM9fWDPtNC+6R!D3@$0`dS9BaMxVB|>160Zks5 zMFs!@xKh5e0N)w-PohyqD*o!!&^b(Erk?l%oy)-n0-V*jEJsCkg~?5wvugPWPjy{q zauaOaR8Q4WZi4=X3e`yQ@$v0Dc0<{N;Fb>OpuntIT1JOX-BBLtaly!jQ_>a2L698O zq{!qT$lYq;UGo;L+I8%PnGJO|9R)Q}^0+>S|+< z{Gya;nt}35Vz8B*ypFAN+QOe_X`S0HC$~eC7stmZbm-KhY18zu3%;(UKrM{}t6e7* zs!PR-4{BIDuc*9_Z1bE0JWe%v+X%~Pld~OQ^2qlRr?~cto#cg9usi+o=U(H5EcRl zN{!`F?~9l?Xy5AZrf67MSwu=rE=u_UAeJhqzKBI+T5Ax*!d3@a2n@WZVlvO_NJN}~ zMuikXkaF3tlu}0e(y`QC`p_VNeGAi-1Zi_>Shq`kJ>&O6y=G3{SqUzHkS^6DCmsqX z@5fMkm|NaW3BIQSCKi49l0)B9Vb)jyi6^M|Nf1k$*7v-E3P8u!cT>W!q@ZCiu!Vvz zINT~koLjMY!+`9R$di$pO;hL)4F;m1!!g}w8m>xV0n8$P8u_m1>m%b^DZ$kU(BMj% zG=w9TGM~O%S>qn_Bg3Kaz*2}|h_18Zqr`_bK_varzaPi^={9xqhN&N>7zjp2FiQ`8@ zN6W$%n`zW4Q4IR_A39?6T^rY~EG;XwuQNb{hK(oQJ#+kB(+3P1mDjdIadFXwf^$`q z7MG^hYcOH*jIQ1K96Y$EthCg>2v|~bjS-_KO`3Aw&=C_lcj*}$8;A3CJ0~+^tasnR zQ>H(JSN7~Z5Su!2>_~ZeS!FW7iBHS*sjJp=^Vgc`n40s^MNq!R`?#yq|m)4 zF08BpxnNOow$jbS$k1&*9)ln^!MgWlya1xyPqk{NJpSFkDZ6pw(7`=qKx$run_TCn z)8nua7a@%y8KJ$qq2lrkmI15Apgm6t|!7WNWq&)c8k6UD9Z(R2|{27*eAN(tf zPOe+MG@6TR)voi@_y4~1M#;egyR8>|T}v@=$e4jc#;#iN$<3P;P+g)cm2&f|FYP*X z9XoO6hIPw5y>UlN=Wcz+PPnI}q@>{dY2%|tjho$b-&1Imo;`CC?p!g?CZdeMNRVf{)y2cR1uP#Fz7+0AoAP^OF;AbkNjxPY{;4ak0sfsNW37=9}Q%jBbL zKdQb$+GpBSD{VhiikK0P=H|0p2GyR@gQ`)zea!|Yy?~->ieal6N||P9gXVJ)t4Z{w zNe*cz9J4TkEOcvCI_OHc$07-0a2vC#342t%gi&UyI=UE^0cwK{i);ktEuz$gIUJ!H zkAd_Jb^QjUM}obHvPf&el#YsK<0?PGpuWZN z+gXCE5!`DA4z7GeTB%n~_p%%auYz>NIQK{RyXCcE>Q5BWXcWB|@0UdQv)&xJecRY> zbGaQ$_~%>rm;3v67Jgg*Zc`B_3W>;53_9i}28Fe!t|WyZ4!(~<;=)#&gF{?w?6i9x z?$o76anY5vYd$}B_Eg7C-5z=3`zh|z!Ooyah*dn<eker0Xq?r#qnbkV)$l(K9 zx2(fV#TC%^aCd2Kqj%F2n0i6s)kCr+NxqgVgSg%{VXT87sT zA2qR8-@%4F^yocsj}T~qFlBoD!7^unVaJ^NsH&~MQ27~x^^ zTj9G!Q|UU&;5Ae4qcHU(pnfZ8`9GbrEFx#aHxt;Nz4Eo6qCRJJZc?i6cdoyWLa8?h zY62P75GUI^u%nsIO(C{m_2tNOd_mr69P}9iAa@PrVBmbA`h=VvAZ`L65)~Ei2)_U{ zeg$|3K+ZTaXu>ajBwS9#;2O;-az@S?!6~QYh+QFEF->v^WKK?8RT0`mYC60VOK=$Hcs0I8Q-%bW|{ za!^S}eSM?&Gb*=58t&*y-+6ich7|w2;l!Bn zy&zB~jl?Vr^WeL#zEz>DWP%|gHYt8MaLC9eO`FZ1`})4UyTI)>E4$6~84vaCKjh;N z7l^apKRT5k0;U}x^sJ77*rP!lpi1>kfOT<&%@=GHh7dQUY^gT?q`tdn6cq3&zA8 zK>A?ZYR3dH2$uV!X4+Y+oEQ=fRKA3UrLRJw!EnBF8{dIbUIP#fMmg?V zRD|PUI)~osTT~i0yjd`;aG$eeV;P$X0{TIDcL6;BLJt5V{tP2N4B7jfcrxaW*34Sa z3M1Rw*wMgET_oosu@r3}{$ynw1hI?duAv-cVY~w(QEJTcRvwvm+{Qcq22L1nC_*tr z#V-J+^G>naTNw>2-kJPj8o&G!8y8ojX6?AR$}INk@)eeWL&svech~lfP+k7aOL=+O z&h4A}4I25yijPZ5uEU)#HEPtld-@}W!zBCH0Yavw9KVt=%06@ zeIV@vjk=Q9D997sKPdzbo+DxjLVPRYm(j?wXrJ0F!4rxcvHpaaw)FDU;}Vmjo!ZqT zst!VO$SE(yO~;n@h38^#Uy~ft7a2135e&&8SlFBbGubZ-^hL6FT3;e2u3Hzxf1&oR z-QV$~$m$>Y{YqLW|DNHOSBASedzHY0FZK(NoGB+lzEQw7^jBGQ~ ze!vV;**hV80#R#F2}-~xWfl!a`=tU-dn8!q3|YglD9 zfMRCUE9;u*A5fm1p?`F2c!eHj_V|OY_lUGJA>jrFp$3#U2U7mEF#yq5Ok$de2QeNX zV^j4+bnI~zB}AY+eTtbCa|;fPGcpEqmyv@(pI30pL8N*zW3dweEcKQvx%E@Mo>#Fx zQu}0tEM$ZsvkR<@zG8GG%@w9A9|fnh{A-Z=x`coYi%C+ctNL%KaQ&}}1?NEixc3LQ zwdh^=f8UVu4f8N8?9>3JmISeqNqqpT)7?MI z>!mj6)qCLHJv;YfHbvMLhYsv{f5Dt{=T38>2i54)Jkuty|}I>eM|YB~@r= zkoG9zxY)Q>SvhUmbf{6YmU9a^BU@%a z8+Ym2vtgqqAkQ#RXIS03^)aZOn%V%wlyG*rdF?xP?%t$nT1-rg@3)B2YLqe4W1*O_ z78zMM2E=vhrsC4IYu~wcN?pwqcZY-FszyzkcJADxNt0$VEQS;Rvre6qR+-rei3y2` zNv&Jwb?(yB6Gxm~ab*Vvl%PR_nZk#3Vy{Sxii`v=%tE zytW-uQd4z_^||CpT`lBDW=>*KGGgQljv*s68)?f{4$W{WE~%0<&T$q^n>O#%rDwg= z`l4R*bK+uSGqdwLb?%9*RHDN)DrKf#y@p-8_dT$0*I_rCg5$q_{o0DaBSzj8ln2I)*arvcuTmFGm`unB6tL(1 z`17*T8=boJkbJLSzY%Um>&Z=Ti?g^g~wIw`3*ew1U(a<$HF*R^{esTEUF>gBcT*ri*arp;Q! z#KcN&f-(y!XL)>l97iUe@0fW^OGk*#m_nF zx>l|8Py-k{I=w|^i)CST3raKq^**vX9mu|fq)NcgJ zXLDxJq-lDqRyhchl4|C*>4Xz+(kxvxt(`S>>ow@qrFV8-`(#whTDH=k1@7_eoOYeM z_O4&QG38@YVscjNHn`@oF)_{4GP?KZ-?Vv4k9WAjSy^qm_Z*1R#908njYI8}l-#@y z-Fpp0-4&mhpmrt0VW+jq%tKtePTh{3duHc!Ai5nSzmk(`v})a^Sz1f!{UV#%w(HWh zN8e`6TRK`=WgLN88X1|nO`17B@X)YP)6DEPjhmQ1_JI9%=+vWa`>wTWr34v@ZKBk5 z>D~`F0z@oH?gR3eti|1zmfosc&jGkZqLG-a?6%J0w9JaN<#muFoxAlx8S~D#V5yN@ z69?J8V|UC9g$=;F26sVxNF$Y&$V-8r=H_+8S+vM#9iL!!fqEmYMP|3219S2^T5mT> z+jgA~9@rI(zb7Zx!~x(i>eX*(z0tZ>xGdh{8yC08$i~sqn$&s?kekRLWB=m4*g|3w zng&TY1!SN62xS|u@6w}RyY^l2_TZ3QA<5|GtW35XjIjvO3zEMMNi{r~K~5KlWowj2@WqdR>>hQHP+De#6`xMgaoJArl?r5a=~Ry zUfa&tWAGPTc5dH*QkU)ijK`NLUXPVlUb|MDo!hQg-=T1ai&ZM5eOn{BMu*Nl@zld} zK}8wdFTct{X-sk6p=sJA9nXuF8QEw905MFkWz?mKZqqr=wi@7{IBxrOQ|qGwcMrz| z5?N!?4zE$8CTfHB9lF=3QCp8tKyr=R*fNTrwD$3nh7FqFsT3C*@3rrp+L&X`%*w;v z)T~)LYAvciJss=x(xH2HZd=^lOg}fh^R#YigRb5BA>W-}ts^$M>6=I!%v-lwwL5ge z!FNllQB$^wyQWoEUTSK6sq?*4GQCAsivXL75jytl z7ACNs+tB+GF?89lZ}nMMQ!s>NUt}Ip^rhnv|48=YVQNKylphe@v8d6kAnnN>EmNf| zI*acHN^aHcL?Hq}vuFV`R7sICRY{{vZ<>?#l>w(H)BBZz!brcm=MGelsbL1OZ+S`@ z$R!wAPhaN%1abocYd*NXFupGa!aFlD$lNVZ%!B>POt7+tRlmw6_jMi302yAP@W_Bl zou07TgkiAMgY{FeY@V=szkG%D(TtE+N?FygW#ud36z4SPU~n7)SeG10 zsiX<~rJr}>`ac=mLeX<7x75U+XOYwX7R~t^R5(=YQOarIi`CQb2>-mq>J*UkQ&JGVGf5So!IEW8eM%NsSGJmp^b1q-K69-IBf zOE)k+B6RM6fy4Ur9pq<0I6ri7-~6{t%(%NTx#D(F{=vI*ii$2znf_2sFUJG0Vci#tKY8CfVla|8cKqEK>yd0c zdE)4tH($Mez1Xu|?wR@M(ZdHXTsYsm?;!m2laCghKX=wR1u^$9^T8)CUMzTP)(h^+ zvBASe_vtsp9pO!k&fu)yedo<9g_or_;#~&P3sSy{!qQKeao=M{49WU+LZHdy%n+xzzH^z5p39l8t~F@D~h zSI176Qlmz#YuB!B-Lekvn{of+*z&1U#|I4=?cI3%_|Z3BeHJgnbd*D+(6Esc0bT+ElWb8o!-N2DHS+jr_Va?IVcUiss=iTB8e z>hWWTXTAK#l9FpwdYM}uc=-E!_iTUrjpuwm0%loiKl0>HckkSC?C3$lN5wTPnEsqL z^Kp@dg%ih*y!OKHN^V>S;rX}DYk&9DhvarBC^+}pi@&{mslXG%;iK+q(JK3mm;dn4 zV?T-$6WOqK#Rm)C1QBh)G|&By{-9B#rjl99mn>ZV*&dHd`~|NA9FY6Q|HfqVdg`Bg(KXxW7C2CJIH!EgF@kX z_{pEvO-X&_`Tup5Wz5L2lLibPqxgH|z}vH*FT8XC^mkLR%zWs3sNxVHfAMeImUVOA zd`^Daxm(|due|W<%NGm4RxnQe=#zi_ z$)b7A;)vgPU-kLo6-z&S_=%r3Q;T!()fay4ERI`HoF07a#|;HX%FD`Dt^D-UkLJ^s z@1Fh$20dfkfNaV2>kH<-Y;=uDni=^4f4gJLI$U%7CkCk>eDp_+n)pZ6wd+^s&wg>= zo^9SgF_|}N+_c%R{_(D<58`gcko~6hE3v*|<7U(DeL}Ji3(jlNyKj1L7J{3zx$mLx zNg+9~Z|9axtEbO=a?Y%0c5K_=RaQxmu(a>kZSwR-G1~5}XW`{bb7wz)^zc4<&E#nh z%d8L1<)uIU&zaLFo5Io;@uC78aw6C!9C~kAiiV8 zDiydpV&mgy-1n5W<5u8~d*6#MG-%ZHktcul@xr-yY+*LVYYGp(`Ey>}yW5dB)KDEe z^>A8#?diLwJm9sw(QP@kUc(0;`(eX|M99DR%n@>8fjbJJYj0m6l%PU*)gP{_En8${ z-v7||QDC-hT@S+g>e+WlpMFFC^WT12akbp@AW$a{7&uyHMhgngzVXT*3kuG7k-M1K zxEc37i5yWA{QTo(pMJopPJZl1GG~18;`zC=U-U3nF-^v)|Ki$Un4!TbCpiyxpp8+n zdE@H$-kHVRj~{yEhq&e6!Y%A)3!JAD9((dV92Fi+<2K4jwuw-qeL-1B`=0mVV$MkW znuj8CbpnHh+SXL`%PmxJD%ckQm^mIG{*(&wNuF`SKa#I9vrU@q4J2=*`D;1_X7oj4 zQ^X;W&vXN`Q`1P56wIa>X`ku6xRk`rwO+V}rQ8x9U-amI$b(|eEtF(ZP0FDX%50%s zPO=WvBhxYyMB4ggT@dN}-d-H%&U&5J5nYPe24BVDjV*p_kELhu$ga$Vci?pe}()wsdXwE3O0ugC& zRqb4$oC-9(XEdDO|NWgr5WPk(NkkBBMD!Mf1VIobdKca3z4sc?JJI5!_uk9sy_eCA zI)lMrl>ha;@AdO^7G|xP>zwo6`?dFR`0Yr6A1nPt=k1r#uy}Qc)02m1Pb;`DIr*O& zbRXm-i>#&p&NlMFlRxO?ni7&%GyOSwgq}TC5y`IFwgaKaJR5@_f5dn>U z+jobH!92dwQ-2hQmfxWMS1Gur|6O>UyRT-zJqKJ`%d^>7l%ofunQyT zS0A8h36vSAJKk&SuP8}}#xMvN9ikEABun;R73?E^#w_T26rG;S;Mb#Q1#M>cTM6kG z{VuIYe3$EfF0ZMXQ&8w@Up%xWu*4NM{+g?N{LMDQ=zZBQVcfS~OMbLq?;jB(pzjBl z>b{v@@&K$uS^@C?tV3@7wvD9@El@^{asO5j`pWP#Z9dWjqRhtGUTw8&(du^cPp|c= z31`Oh;;_>irt$5?X6`K!RV2@T60n*(t-QNnLoT~giJD&g;~`tu0cAwhiNwC;tA_^O z*k_7)-QLcX=4Q0S&H9T^ln`}V3 z<#j#Z#PH%-3COq1o+LkMle5j4gqMd&cAd|7v_QbXZ=M#AVtK#YIPigr(o`GuC!iR>aO<{rF;t$h zFf{lMWA}?tznIztW#-LE|*K z^1P%qr;oQ=e8Dt6B^PLBLv`I-wFimXV}f?rnxK`|oK{>{>wU=d{R|)N zMg{6@7hGmq)L$JRH=MK>UO1r$qw}kQiJb}ZP2A`3`kXmRDMvVM{*B3EIF4_ZjF#R2 z6=*4PNCC72e#sK8OW!^8|O@Yye%Pvr1L2=ih8fE z0gLnf?mFXyM4R6_p~cseR>N8S!!+Pro1BykUv4Hy0kN@p>;^%m`vhU)qEp z5XDmQdO&r!wcCsIeZn?U-A)$UT<1uC2vV0j4k)Git~0Is9V{78Q0m2tri)&7bBjNk zE?jYz#4b}$7wK7yiuj&SiG*%-YU?_mcEK|nc(>GYVPap<)bEGR*VpYN4UtnR!V^-$rqy&Z+%jc4~43c6j9%gO-eoX2CY-{sxR zqZ38~mmL|-*;zu7m>{a060{UlOC{*;90v^oRqox%gG`3x_!!(T!dwpLee9vuXcWR~ zGFJt7nMWQDIzsk;AlNirgZdl|=7|lq`eJ->Jz8RHyh2+!oF=MfZv#aQsP^ZZ)8u2@4~9*Q{cp2wZu0xyLE{<;M$*-U|gf|AnQte)C! zp`A$d{QmJKt~0(toY1Rh2Mzj~<+o!v6k|R6cN?C6o7!KZCbAo1;rr78PF|r8;abBc zq;56ewaDSr@48z&Gs%Z2daY3VcQ(}AJ!n4uiFpP(+3aT=Uup5M{aJF`PxuXsh)VPj zxSi7D;rL*cpB~Ki{X-nR*hYw52ESq_?4fu(M+AA5F_9zUC7iE*6rsk@hw1%5^!Req zE2gTOGh?136(w(M{6Ptqk~j>;FvX|;oA!uc?K`VClY%|(5_bnd4c!4 z>}GR(Zi&8hY;U$&^|R;)r4DuJ0J=!HXp5>lj&vr~O25CwS(4P!!A=R$Zv9ELie?)= zuaOUSES<3N8>?o;qx^f|dTfP&8~@GxcP3g+Hx_s*B1T-ct9ra(Hs1$?1kx_&tjmp* zbSvj(%`a0lK8^;z(70y}#VFK<(EVCP%j>rxUiQlRyLj;`k}C8PY}R0>Pu6R^>~<_} z+81nz61xEoedW&O6%+{b1;~+U-z}4pymn5Zjx1yd za#Jp>GcZZ)F5~d%Fg?iBW~DSI@}g)*$9%6Vg|%3Tz7?GRoNQu3;@*i) zbUycoay#lQ2gU?0NftrQr*(&1$ zaO-G?Zh|^^F{8Nfzhezkc33&|vIuFk)hF}b6t@Y7=N!wkB;s4_TqPX>1 zekZK>0)wKmH_G1k%)n1mA8xAqPTAwhj7g4fn;|SVMsA1?OeGzr*o9*U^=IK5{&sxf zmD6Q2Jk88_yx8(oO=PVF)rxvRT(<{bb6=}i-QSY2tf9~THoJeGFG&zK+z5w@Ve2+I z_%hpVmdaog9uE&$GBT!U?d4=l7AdkETSpY)L;vQ!w!qsn-o3N|B^%w2`7*DQf3X&336pZ>cxz^ZM zmu?2xIcY3eMEvDG>!b3HxZ;ywwV* z^X@e~P?q|%)H+S($XLRF5^US87q_*f?YHV%AVsAkKDwJ3k?3C8Uo$v&M_O~NO&rlr z@*;Z*zsjXMIn8yRmrnOwc5<%_H@0}Mzzp;{pgm^d8d^${_>!ZANz|bsoUA~XlvcV< zl5vVLE1mSI82W;d$4bg*bC=*qLyx-|ny=Xj_^|Prx)`aCSap-dv%z7#VMHAF7rlXj zPE-c^)i!6zV@H6JB4<5Cy2_o=^Y*jv!)6F#Ld@szxojlBY&6(iMqbgk3M<#(WzpvR z7KDwZDAS1qs$h(^9z*L1tU)`6Z74L8{c1x*!|vn!oSy2Z}oeXFVNATL&~mwL7#NW9fXvNqOFy1K`JUMS|ZH=D$sLg{zGB};rbXAMG> zYl3KE{{~vV9u<8ZCz=)=7iG}ARja9geGiKhgz-W-NPka(gZSnxjJ<0t zGLl^rx03wj8=0qFgXHFF$=o;BgYoy-vjmKM9c^Xa_L^XDJko09I3>Y)b`q6~4-#A)d8Oast+JfL5XVKS-zS(GcJFfK zihIUPu7UU&G+OA<&@^BL{fo5akl)K2IE;>Z1S)Iw= zQ|)Pd$Q_X0eHtwKwTNxC+DRF|ErPX>RMm75AOi~(R2uGIxoS+x2f*sDWqh0O&ke3#8VozTE2t|m=i-jH}V8zgM zqQ*0H_{;l$amyY-(@?nP;-Iiiu=nC1PO7x$Q49=)Sc{OHxJSE=rCo{+8~~?>@EnaF zt9`^Etc*X_^B3>@jgRpyQ4O@r3sprHh=5dVAFrz@LC3^^{IIJr-xKQ0<*wKH2fP|O z{1x}d378OM9C@Zl4AZDhR#;LF_v62^x*b`|q45)1eP8&(3V&r1A=|FGjqcsxl$(i- z+rvsF9iUJ0=-NRkkB7bL>3ZjwB{ojBV(5^Nqv97&=Q09t7|H zIxf4O3P!Ef9W4@I=!X#<$Ill#Mux0;>~Fx8#l5kamaiL~&(O3Bm3^OFqci{}$R>Y4 zsf3l?YkN*&%S=Wl1q_fvma|PiVRr3(9|YGGFQ%}ac)o@DZ1#Anc`ril16uAA!qQOl05w+FWUh z7|#|z+eF(3pmWdAh|v~@O}h7EAqXKNL~&u`d2;#w`L2NeJw|8-50>7KaBWY{NL;&s zP=bu>$gZ2=u0^1Rp~G&hhSwt21h|p&Bx)(qbhK#pKXq7$#C;rKe^P!dtw%mUYM7(! z4Dad-Owsd~sp`u$MtmtSayG%4BTahFnE&u)E^aSN&+`j*b_5K3Xwm`~486U3TV4wO zxu8gc7Ux5hD_OZSE~gP7Zv*Gu=~!FZz4jZ?y%7<&BaBTxGf#KH4;#3gSiwLltsVw~9agX&r zBf^>25hh{c-Uv?obua1{Za<`XW_T^;Yyat=2Hr8DNxYPU2UXQ2hlj_d3#;ueRZRup z9GM3S6A!q&^Ef-!^LDzOu}<9=bAWo4>~s6n{Ega?Kw{^NI3NJO6c_=opRCjp)TW|Q zc7DhtiNvH7BQf8|(y);0fPBTaI4-*Nn4!a{|2J;Uw)ntL6NX<8nJHQHVI%Ce<`&3-&a-7kV0Vd# z3dJHl$Fs&FaMB-4es6^gRnKYSG)cx>6%FIw$(YaGXPA%>ogg`vc9+Wg?F94wMkVb1 zjF4)gj|@VXPpaQ(*U!Spb^7udZ#l;SlY76&2#`F0MCZizX7i#wW*;@V43?|xc6;0` z@_o`1R!F!lr#`egEr_!x`&{3g$EjFn3CDLD!r4;`82LPF+|28Thlj>3TU{?z6UDw2 zWQlm4_sZChP-fdaMA=8o{Zl#0mqCKUg7JLs7V{d{JxONLdS$!-^Qi zY!KNQP2eW)oGzDA?-7FMd%9wNCzrdlwhR>2hLK5f!-{1-99FAwX^~9xb0E^}zqQiNz~_Qx7@pDh3^^T*y>? zq?VN7_cuY?aLa-16R^egjwi_DBQUOZHT)upf z=X2L+XgxjdYb2&f?A-x9ghNU!E8eVV!0JbX=YUlh7}6(fQNCJCO8PT$-vieHoAkZx z<_KxSDwgA z%DpiUmz(Lhb@$7qI0;>Ul{FO;7FvGh-!n9&ZZ<2Q`OvW^iuE z!T8$Z37x|>%|&Y-NHai^%joEgDU9&dzXa6(+%!{u=l*wFOQ~6;#psB~_up_qEp`e35=;wB0>3r=om?qu(Mz>C; zRLT*(x`d&~RqhVd<--seq|N>ba*2a=;1=PxF9L`_Imm&LU9-aJ@OB3%4(+BVf0yYW z>i{FV=O*)pXR>t5r4wCXvf_T0x8N<77suSN=GtPA1{@XRkt-d`p7H0;mK+AxIpCI7 z2qs*XU;XVF0gM$XxZrxi3GIk~+Iveubc0COff|d#8scy>b`h*=h<`->(jfla`^@;M zCxWMX+`iN>davt2&tg3Dg7YhdQ+`{tQ0G4tN`Lnc56e8w??O$DZQyWAqT}^Oud)9M zvuAl=FAFIkvGiO68;QTWdc#mA4FmUPFUF0HGd^0n9$#-NKn47p*8l~OW`QyIGiC|O zOylM5Zch=48Jkz&Y&^wZM3AMCJ~Y`8xol#?)u@QJN%5qTew_Y;#U*B|g0YHjK%Gb8 zO7jm7XS)AJ#i-&ysooDWqr?vT8QYs8PJf8SiU{(H+pX%ME*J%Soa+lBFh#le2{DKP zkCo#+m#Gq^8Gsjz4G))Z2XklL*sZrQNSk<2eR87)AP7S2TAvlHHfrgx7#ab|P44$C zTbzvgf!*Y$0hMFH-l}5rqUD(HaCnm6yUdQAN2cUFu4`sL61b6-DrOYYOL$(N_v5L; zas`c`g>o|&TCj!h*H@h5znAMyjy>`4dp`;m*y_xCNzHJGVK8lc0jV+tmjK50MTrS3 zZDK9@>g4&Bq){X#J4CUq7_eNj3pQQ5bYp**yxMy9m6p#cate5h0_)Dkt{nyRi=AXn z3S-mS*C{Q#r>`%QM3jWDW+M_?N}i<4ZGy?1yabOM=vzGbhWM<2Rj%Jf_Q32mG-35Q z3uj%Zqti-}%Z9HI=@u0Vo9aA{fx}hs=)@5QsrVk5&Tq{I@xSF?*I{qmo9u316d|0B zD)`^%KH4*zxWUxBxC`t^Tm2TkaxCDTN&N#I!_kTzS)4;BrDOBazCow%nVeK|)X(nR zV_7yV;w*asN0{U7D0p-+$)nl@iw9htO5m8NZsl`;+~y#pTO}CQMSb~`nt3<5Ez#|N ziS)Z=#HUk$W=XiwJW7DizeUS?wPG->p;5LLJ@j!kxvZWhnD}Oh z{rLN)jl)2UR`9>E^v3APJrWos?auK)HK0-O1523D<<{Sk`aVqwKIM_F#K~+uogL1_ zhpTd0(9Lo|2~vkUM#Tiu2md_8rulTYTxWh*;ANMO4#`2q$zf^osT#SVu0XVWZ;C!H zN3k;FdaIXpnmZChS^68kfA41)or%I$qSS@%$dRn) zu5?!i2{&UdcEg%nIkcJWf}12ZZ`uM*z=sm zS6@?=WrP7m4?w=!xOLwP`eB7h8dwZTc=|{7b>9BaGYjzy7M$^N_rq43?GcT^YX?O?`BsJ4)!vwl zqW*pe0+2l@HMH#f+wTrxj}{B#is3WU^h14#ZKBNdFWdbnl|c2`NLm)f4B^^OGJWa} zNw%cc|86+?P*2_5HiQOcIHs34MELVgHg^`N?KbJ%?HZ0I$8>stDbL(|ce5+gDdDrB zq>N=iHu@7&bDhDuCs=|>Mf*1-MC3b(n#Kv5vZVgGhirhZ9mdhS%Fx0o*|^89Dk^FU zCLbOdxqsh-*PbVn_`OPd5~*H+7fisRvq#()nsuO4@X4Eu0MAQZ`^86*9=R*2kz6gH z*PP*iQN?C#|98ief)$#bOoSvE+a>zhG-Y?+ZBjDG|G)V5K|+5TCSQY3Z#66!aCX$n z6kmCN_*DM=wF=2|75AnmeXZDOCC?H9!3nY56U6?-Moq)aPaLvt+Z{lTq*(9rFrAF< zqQe)MV5@ynkAuF=e=Gvil;Qz=T=$&qTPP&3){Hpp@UWs$R zp(d#@OW@xJDygah;>g!DxTM&aioNptT2FWhuqm>1sE9+G34$VCdwN^D`7o!-4E+E5 z`!KM-k9U8Ej?^Qo)fuIA30FN3*y6voUyRqbE)lIF% zKb!;1=2gZ>|93H2WkcaAWV?ubf8-eHxPZ8T-Sp^CUlV4~pIq_}Na&S=FAW)0i|u|| zj#G|+Ko0u*JQX=5rJC1%I{Ix*0`FV0}?L?Obbvdno zZE~jW3~en$$q0zPJYJ$!bBV-n#iBLHGbO?TY>2~1pmj#~l!8YMCn7*5w`5;_GoA^% z!G0|~jW%~e3n!W_v%7b|(7|tF+?J}V-knL8GizNS>}a8)g)FW=-Lr%s;m;pH_a15Y z@L-`h1t1sz$Mh#5Lo9r#T4inDr0v)Jc~~eQ|8MAoBjuWpJkImkD`x5}Xvcl1aan^= z{s2Jro*yAXOsDI&M_q4?wJesS!*~EoFmOpAB)V|kIyEsfI`6Ub5oYj&Z7{ybZn{F> z$|o9{DdH6R;b{i9pj(m5$Tz=WnoCDVrO7I#wHAT#e?C z=C0TNfpK5I{Wl{vWs{_w#JG|v)61acOaoSjH@Z~j{ik2#<^Q{o5z}t0AiHmKfO8~a zOJ;R>HZt6w4!Brr7@8)0P^^kI+pUvv?PGcCOfn%3VeT`zEA%P0EA7;ZVvEVpicFcEUw zBj4Rit7YjX@C`t(bOKWDa^2B6_ah>1K=q?_G8g2#w#IwE=I3mC!60IaUgNf}*s@1e zc#SSJ>Q4`yf)Z+4-dMl4gK{*xzSQ|`fi2i+rY`kH1MY2~?9}7_7)`vj-r%D-iZuG% z>Uu(cYqEEKP*8Jzo?+*#MRoE}K7;Uiw))HQENLXs&ALM?Fro$6%IXob7&lRy$72~9 z^g>O_*@`O3Z-VmRQJC3S()U4@+d%mH{>V?ln!|l294CpEnOT;ALEs|CIhslpGTX#s zgRrjo&cTy0*JlP_nqfJ>q{3?d1U)%@evOv6*Ssa~Z2SW9_r00M4FCIC3%9b)dc9t+ zn8A{e?5OWZf;4vOO2FWzx6Ugk)fKTV7^=n+3t#Q994X>kxS-_wB`5FdtAgX?QUfr- zp{)l>p=+uK+C~npkoil1**@#IEVT{e67;2Jg|r8K98`4ZRD-u%GZ(x_2(UX7s!A=Nf{<+N&FVOkCZ;EI=hu3?y0Fr$R9H0LHLwSS!&qiyA z`R#x*Sg;dhAHXZ@HE>tMw~B5eqy19SyIJBR`uP25)fLdtBs}g|&)G#9 z??VC$0m>Vsa@f}RJ`!nHem%#?(WDz|95I#2f?CLa^3vCVpnbYpVInrmtEgx`#el<{ zlXWFZ!rKDCj^1RW=A#M=1$ruW#<}m?uJ3d1j{w)=@yzcL6RyQ$zJqAWxic7vwT4c6 zEcMStKA_ejYKbx%t&|!Ni}WtJcUzL0DJ@Rm5|$n41adH8kk?W7wZwGaJ!eksxaXf7;4+&IZzhL4sjfoh~MYLotz}kUG(fPAE8*z`!g5B>Fb#mn;+jj_$>ll;4I^rz+ zl@#3t9dYO*N!Ee4Sh(;~p?@XQ$~nRT$w)u}YBkj7=c=x|lg)6=%5ri-YE#?ul?a}3 zCJuNoGbeKvep<#iHlBL^PutFFHa(O(?_y9Oa7KLBR4bbRQ9tprFgAt6d%I(&3N*aG zR9^O>N=g^rbU6znpNQgjgJ8Hb#|mnNmB^T2@`qBPXb4SDOL4k8s<7fM)Il#XM z7Y*a%_G9fHUJw|bp7_6N$hqa9%eXklWLb8E#{3;F!m3J^0X`3NTYgaV^N$>FV5L_6UJM*q`}KTN zB{qxnft~OEDsKL{K8d24iqa*Uv>o$;`*1MTUIG*BeC5xdKYqBk7nttsM_gv+4{YE_}_{sX4cG52u4aL{5$?^`nD5tgb-JmZz)TX6Q?| z2asU%tNA4ft3jzOSVvR9T{ z0<{82&4mR?DMChxJtLGUzAaxxCw50M6TF$>sMM4e01*41@Up?dwnqI<0-BE(np77m z&3FB;w2h5xR%t^ebTtl-jmbq#xNst3D;xsl;j4|d`U0kX?%yzt3|}yL9xeF#)@~Io zd^g%;9u?(GMFSVW+E??)I45W(Y9`_eJzp0f(;ax3{dmQm!|5x~s8|eSe@;Z4A*%zF z>=b@iL#>i)`@U<`xTOW~@T{LnPCybzAqw&2ZJ zBrvyZx+Ve{|7bdqOY#rk5#dqld@Q?#JnIzr{SM|A!V@%Le%qS6W|Ym_I?TN8yd|JJ z+v4rajbPNaYqAa}&*a{O;`xWAl_KIZ8jWs|qYm0p0GZvLu~tpLD;S zCwpHr`-b|TeWUd|9p_k8$G(EwWUcs~{8tFXX8@EW_>Qn!Tc#1v1Hh@SMqFhsNrFP%kaKh1k!e*up6XAhP&fGYH+RwB*o%#h33?btUk^RNnF%oOxV&(7HF_!F)tH3=gP zi;NA4dsS`_Qoo(1KFSP~uY5zY2rtd4B9O09`;W02>6SpnK|+=R^S{FCF(o%Byy zjEz2;G83$*8j#jj>g4svn|6;z?jcsK(D7(h4&y_)R~b9ODvJCc%XAE;X9P>;PRv&uX@(s(b$L{bodq{ntOG%S;CNIId&sI&|L69&wdsFgG6W{2y?zXhkYsRe zvC;?F%FE0zb6gae>$TPyF5o(RekNF*JzY(yW)vuSxk-*}PseM+QXfY7K~Qkrweaub z^s<^S)9+NKCro(~X4`?@{194_Q_o8p^M>)YW*?*%p1FcVV-@O&K;E0oAJZ62o3eL^ zwHy0^zCO|OJ|Y&S^w@v`Bpf4q+KR13waK#gFy=_ph;%C;taGImk;}bmpng^;mFYBk zKSg~P#A{grT4CCJC@_T!)0 z+_h_SOA>6S0m3@hgIb5ySe8Rd7MrxVU$C~5PZlniG$$@{t$Df~%v(IHE{tbMC@@$3 z+sA!)NuIghY{HvXrR|f+ge%8{Qn+ic_P7K%Mhq8>Brc0sOgn#WUUpblM^*@K76R9R?C?$(`yC7n2{%zy4oBrmox*#!FU)nE|g2(Miw%B3Y zI~h;5TzBAna?2O}+s?^1V4m@zIzJ2`u2wA~@{f^$oFo{NL-BJO#Rtjc)dq=-E=re&Bm7`;jR*X9Gc(_}lU*Xq6 ztK+HbaSbs#o2A9k^9uP7Y90IJ@|akHZwh$Yry%1Y{LH# ztri3MZQbCfYVC}_33!VXcTWVeSDJk=>=FQ2FjF+G)2x^B{I2JWT)(5u?y?3L1n2*$ z!_E+QijR^^njCsmSI*w*3h}iYDc~=Hv^%jcSHke%?)dsu|rBrVNAZpmFs@o55EdYew;b7eHXSbgc_AIecC6xm`hSoq!kM%+%*yp_- zyF-WG$@lIX-V+g5kz_yOq|?7{79;F&3`^?`dN4po4*f;yHDtX~VrO-a>1420_uy2C zh5cO~IwuIe`{g+@pBnI6;uXn}FXC_8MlvQ20qC2-CgZsAbX^z^b7q7`tdGe1_Ufh= z1dUP9l*;MpCa{y}qaMMwH*y-NFW{|lRFT;}Z!uWH24wBjkGroCQJv!Y*Ze=K-!7E> z)Ys4;&HmoK13@F&7@Fe&Hd^9tM;Rn{2V7{SVAYP5+Uj|F#@Gp9V*6+w+3t}3V6{el z3aFxeW%(jR;^BWttQ3e*foaA-n^BMRh52mN7m%<<#IK{##GHPu0qj z?nN!_n_N#=7=G6T1Oa;?JfizP@}N{ z(af3uDpTO}*{|hqpQ|JI$@q+I9J#Ghe>Lw{%?7LnroBY69%8;5CBFPt8us3JH=E}u z$*o1)=10=P7p=mtR8rMR)`_($a?g_J6#qW^Gl00W|M%ITDfOy+)s@}NcLkT|v6O3$ z`FqpVYJl)mK<0krGoEI?Q6>|Hu3>;`y*73>I;0b#JXIVO_jU2aqyeMgrM); z;8-N?F5_Xxo8+fKp^*N5kKUM=m=g&Hs@QJ4CweVf{wnu?>Nx-J=L~)BjOIx6sGOQP zzq0*n>*t&11yH`MSQw*ac5-~iijvlkGIjH-IKQij_FH_@j{0Jv7`2hPMv%8Y@rsns zf5$kL+04u$%8Sh;PYp&E>?gnbEv_RzQU4_sAz_Y9%l2^6@1HMTLxVz-)|781H%-1o zXNRX@O$$1LIJB`gBP{tIB8NuqU$*+*wvu-F_iD3{FIFpXE0coy!8@lN4fMh$?q{<@CQ4cwr?*irbLRik zvltj<0WBq@W@cv1z@gaLJ%h0e)RYhYl@ML;npUL`dgn^x?F#xly#=Hz;Hjum27f)|IGmGF8DYo9`-YzMFG>_ z>XF3{&BzOXTDiX+8|7H=SGPo}oKwaG39CEGq z)nq?f9mJ5%CLl&Va7#+ z&&37xgKb{ROiMcp8$`#YJNH=lw?W7xe}YeATd%OvfxA7#;6s_KR(2hK)}N8!etm1tu{4 zi(R7V9w2IQ0>$nKl1rrFv))$ymT~j>eH!Htz27a)h_E9zTJRsnuq!Eb+^g_4GZ<^E zX~VenoG|5)*;qikh)!J}xpy%J+SMadtjUc9UIIw}Cg%^Qd1BxDflNv7))xCIGwxmm zgU?~*S)T_GNHC_rl`)=Y@3fxgxbFz_qoT|q-W&+QgLZvZ$$98y((N@iKKqV`j*3$7 z{}5LHCumVr@N5O(aaDd!pw^MC`mz}4w=8LEo z{(btkC%mK+>Llv--`*ydJ4X0ZO?@&R-uj0Hw$X_;uSjP2hll-6qd|ZdgkelUbtU>0 zAD357vbaOB&?qMoKcPoglo_L701=h&Ehs(G-0#*mt;)_zn_wxSqK*jVT2Tl;@y+=O(Xv@r1==N zbqF4$=-ejpP;Wdh_3d7e=opxB=(5v%4NiahCiR*L_vj~c(G9mDMAiCW7~|PC3EwwW zMT-x741wk3Qap=n!Se}Iq`0PcPe*$|>SDTT#F?{BRXou;6eAjyR!t$hZZ+c5FMg6= z4h%19ay0!%!NsbCDfCr{mWwD@btr;E&j3ODe*TTcaBd{vukB`Z?uLZTYF2JHHam=| z&a&0c@?H~sv{-g4Tev2(rz+Ib3S&ue)MVBxpb^?F!MyP%*J^CIda zUaXlAcP#EcnY|f4rbUTe@4&Oh5h}3ygVbc`IdV6u9YF@UT2B+P-ouCk8amuuPZmAT zj3sA^##*oTop#T6YsXQN=mx^;qpo1Z?eYva;?w?|w)KLaNSp+OWwY(f5ZR4l`bWfM zGi_+3fSnd6C#Rp_h5IvAZbDo$^>9VG%O*F1`7CjlntgdApbAA{L|!>Z5Sk0#uF&%` z=y1-H>>Y?Kh0gVTGc(QE<`~ZsYCpYiPAFa#uspoC?>y^KWO=0uZG4;rhEn)y^^G^MM#R%L7F^{ai8uTh!Qc!;r(i+5J0rt6R#@1|V2nZ1N!XeYpl;HL{%L;p$_ z!$U$+B=iLt3-k!3X7M$#f{A1a)n%bd-}$G6emCbK6r8-ZHOMxL zAC?I}jN1vDkbJj%MNs5-j}WBs^F8PeL5NIB9K&+l^|lNWOo1BpwpmHOO#P=&qSpAO z@@Z<%&&z&N`)i;UfKg0jO z>cQLQ05j9|Ne3OP2gZfk-+79|HiOXo_F8MrI^6iVfUI0%CaVGXcevv@dL5?m(FV^8 zo~6pXPh@6L2(aVD@;cVW&fIfPBlJ)UOhBaR_eByA!$ZCE)GSgf-E}JT7jZhvC8aZIi`;G}!<85cO8uJVuoCy!1M}EvL{<;aq`0HqYok{II&~e9X_k z47XdjYqT4wg*+XBawA|AVq~OUueRpiL3klX-`rttuaU2UF|cC(Gw7VUDnH_>jp8)) zwVtTpY})>)EPd)WUIt_EJpMu?ezgrunz}9De}+z@v(HJ@Ji*!{8z?j@hrYQJ6GQvD>US!z$=Ada`3B z+{QiLw<(}H+_&?BS`WArzOdr0`(W_FfZ)I3NT@oCGM%Da$wKQoDDfm|L_B!q2Y(q_ zl)ZQ@*JvA5@37wmZ*e&?+9DG=A4WbNm6Mud?sc5qVZ_~^P8C0xuk7&w50T@tv(5C| zF|UcPoVBF{Ms9CfG7KV#v)VNgf$cYjb{I46?)R5Rjhib&4)_4(3c)HKclSoX%bR9e zFXQS?<5_ky)z&MqYXCXpe7Cr{T(5?OPrZHjHv;uvTKR_A*O}=Lv2+(|m{e@~ zO&eoCj?Dd7#4l)tqIkW?HYIv=6-e^8?T!DPu0O|>HQ{0B^}bjGx;C0*i{gZi=bKMt z<1#`^-L6mdXWz0C9dOqEPdE;4Heq5KSdFpPsr1cPpqw!Ardvq%#dvA%XVLj^F^c_Q zHGr{>1deFO1{V)HGZx}(l_meZ((Cv!d71rXM%%nXuYPB>c}6j#h=`i!sgUo!VVzuT z2%=-D%wyyLB~fp-{a*qk?xV8(T(j5uND7xPAmVxuyk{O|0jh^>;m4G1G`d{)iQ9x? zj=v1l*Drd4v9sM;-RuImRUE$LSG5CUbfG(=coV98;o&1dK4{lleYxg)wKE6=D!as_ z)T>nhpz*GF{Xt|kmedlGXY)cPv^l{G@i;y|W<7;9@+%2=;B$~LW$vGM6`Rz5wh8Ae z(W1PqCtZ0aYKLE>p8ozkBF!nXx#jVtzrN>9{g%|#y3h+jsQ3K=B>OJo3&SH@_6E>O`YS1I zt=XE94y4U?eQ;A-ETR9=ZS?wpNbG7G=>O2~^c7=i)C*G*tAZTCS2hLnUn)=%&Vk*U0=ko95gmfej|E)lvjGO>v3{cR)jvR*oj zYkmCRqQCR%z3cmg)K!FCBr0$(A!MBOF>1J(6B2oa@1q38E{=tOu`3CGGrpqqp7$_?6S;X5TN*3+*bnYEFOO z9-7_1BPVpLB5wU%VnHVbJ`FbCT$BpXN>=X=BZPZ7%_GU+>EgzT$YU1aI)Ea*-Dq#* zZjz`ZILBckZiq_V24r9&dVV_~>&(N$L&>eLb=g0y&JH-&ZgsG;?Js+s_ns5t7(gUE zdfZh;roSRfCx!Fo-#2eq&96VB{gr3)0eYRTwm=Za$i0yu)XLM4qwKiv?2bqx72z#H zfMZ6R9JAy9NEiSiFcu4MHRYH2nM{?Ec6rF_w&r7y(-E|W4deD}au9zfu^JpXfjMkA zL_b%mQ)>-L20&j=6g^Nrw^*3Nb+?6NE-Wncxh1g6%EFFq)oHUkfw3q1mLaEd>Jgm& zsU#qJC&%oFV!c#;yf$*Nl}3F(-@$M&&jWOEkQzYuw_yDH%}r@PNl zNAp|s4PO_C@&jrt1@bL5()bCSoymR-;LL3T<06Y+i4s?0IQR{|?Y}RNKT9y5YECrP%2UEx!$2*a|W35kn?W7I^}ncvanaFT)()ZOTGk_QO>_0B%W zil!L#YX^o{eJL4n0@LFZJW5=s!RdVJnVM`|JMHQ^kwLB1(iP_s`xX$ zj%)9;7QPW3vt8pSM}24c7899tVs`(rXbs;Ms}A3^A`Ln{>sdvL>2YbT;_zS-9}sESGT{$rV+OZXOFR|Eo6vsMse zfR+M?hqzc{B5o%ke3aZkQ3z}|YlUAvZY{<1yY(f&iObus?n^j)Zb16d>T+^v*IErt z_cK0sE>l*fllb5@U+eDk_>^gjj1=^=4SYYWOflBzD?+?RSu=b+o1bdjH~E(9-9Vl* z0pstFC@4@8LMLR7%Oz+vi$^)`^Z$4{$FR=YsBdT6n2gD`ZQHhOYqD+Iwrld_nlRZl zxhB8+evapPzjXXR^l@K%uWPOKJ5OLPi7AWqk{r=OMUxFgXMQB)FL&u)_k%|qTxA-+ znytRGRw$s1m19)mRYaN?q4*8;YBGT*rL2M8`*HEcnhbpIUn3E2p<4yXHYh5{nOg~Y zx4+t3AhzHC?OnSW2bOY7{^73>N9-j&rd04loWlw$7Mz|Ga$tV91NNNYS&3dh`;Y@p z$}9uB-dn;ac0fZV_r&x*4YUD%4aHo>$U3|PK6W3k@99KTXhi&2s&OhTrrlE;c3|kO zTWl`?msr%k8F>>h;ChaI1Nmo9?X^FX_wjyuyX5p{WY<*tFy8b&LgatL0d;i5o>}2{ za+Trxj4$AJsNb0keavL659XVBRvdy+rsVFcx&jI`EMotx7bEIzB$Gu9ga{KD>7czGpl?imiqy z2gq4)Iwncet~KZbd*)1k`7DMXGBdRhumW3euGBLbV}0rdkJu@#0VcE&L-6Xs6;S(c zox(1)xNeY%4t>0&0Lf?vU$0f@t@#&x#xLXjRA|XA8x*p+hJkn9S9}^RrQFZ^uTQEz zbIS2HW|Mebfd+d^A7OP!SY5Bj@Go%1f-8%js7;3K$l%P-wf7! z{t1A*-xrmXbHf}4<--QT+SDg249?{W(@RfZ=jN#p=oo`1o@D>-TLiHp+Q{z?*?mPt`2u$8zw2WOJ zIN0wLy@mDs9V;=rIA#a3_n$kT1qGo*dCO!G`_!K`nhkRWZ;OZ2$nsfFOsfFp>uav! znn<8mQIM07P@olt;jW`-+?;et;aYPInT|*ug z)yK0DC$Y?0H~Ct216*qJz*h~3A>xR^QhOphK{R#^73cXz!O(A2JS(ro8b2&#P6Za~ zp2{9=AI6s7n1}i+$M6-{xtpXiT5-bNytOt2J*eZtM9t*rbCW@vR575A29yjV&_Ur6 z@L{S{xzx@{WiUEwok~}bd@POblZ#W%seYPXYwj~C)}8m%!d;+0K-4{G{KTT%6TZ@n z(r{B`eZ8bhfC|O3Z9Y5H9ZtI}-<_*I*EfXi*RKIRUskQr$5f!6lc2{`p1`Te+xYfL z2N%ky*?;-5QdV8&Gq#6x!uQvzO220H5#M~sNtIlV)V6=fbcno?XjDRJ>Y5nA_9AA3 zuFW767~(4w2q>5RKxfugA?u3xGNrMZkhup=&uV>M2zKF;cx&S|>dQLs8Gnz4A+%?7 zt%A%p=?S#Hg9Z=1z3{B~;Zvib&^#J?3R+5X^r7N}7*#2{31=t|e7JjjCyDg-j!9be-kC-P zO!Wpb+snYazp>y@oJ3BcUfJykO^wHbME~|S3K*k~oxZ%VT1GeM2hYo+kMV56jS~Yq z*E%g8c|c`t*sBB-4HxAcl{+Pf9ovePbA@DO{a0m_b`^QSP{tw`6c18 z->-oYKPU5^KhH@{ZOY0WU?Q8uM=4$y=zO0cyF_p^2kEi#T`W#JROwaKPk z+Kioz)YMftm*ekc;f5T4w10u+dG<=oZg};J{b~pc8@;=8Ip9Q~p`Qz>=!Q4Yc~K16 zaS;}ei*ia$X@qxbcDg@qi)73r|GwrH*nfHzA@%-$y#SY^@0u$yM*C%r69N&-%F5VS z2Gwluapu6>j>qF10NXts>{e5LUomQG%EW}}=~!%w)>A@F%SbniOTko-)4NJ};?#oPoFASer!MMtBCpqNY=|TBD@o1yI4G-wDAV z4>FxR?|eNiT?|A0OUJPMNkNH$Y4Frm>x)%d*6})r{JB>W?yBMcSv3;EQ#Gd9Al5=H zB|J?(hV4=wPG2Tct6hWij3J_=*v%v$Kxpqcqv)jwby~d6z3#oF=c{qls zpN5uTNwsmkDg=Bf9_tP1Y3ONNE?k#P%uKdz8gT*>d{IT+dzrjW7l_Kr%-`Pyzcl_j zUu9BUy_IkvlVAl5i}`o`j&s(X(5j`VCamF#Ieoih<{v`5>`?3^ikr3OP`tF{Z~q&o zcgSRANHhrLPR;vNzu?E*b12r=Y?eNhH&J{+yn#jvK6Ewf;LMWtjA^IyNJ!yZ ze64KqWg-8iWR2isNdj+bQhXu8)beDjofI@uZH&duXL>3G{^O8DPVXi8egdDr`MByZ zVw23HV?a$?U3|2oqpbmZ=Qmm%A5^9Oq6vD+#UN)1IlxdQtD+odp&59Xo_ep_+9{W> zfK9Ok)|!q#D4pE9j_%m4Yk9T$i~GJ-(nrI>qN8(I5XPkX^O$k%E3|w!D16{RYY8rs zn5Zt5Is>jYj)r4!9Pi9yadi?maL=WIe?KbYbb#0~O3SjOTWjDoOhWbgLk`BVRyP`@ z;p9&f@RTDUTBIS-UnJ5rM#|bVU_Ft`UIBL5x|=PREVhX z7=rC}fG7~jYV|!?_4oC>K~Wqrgr8G}ayk&(?fkkBqx=aDKUw;@&>8DfUfati_A5vo z(qtD+(mZJ#ML#%1O$RSu0f}!2Lm>~o#bpB#mm}aPS`mq8!@IR6eDSmdRZVb(%kSEqO- zuZTk()kr=PdOlZ4r{_X6)#wpbSN1=RD*hgPE3-!9GDJaBW5NRE+lo{r-STk#c~hX% zY6rkx2-S|+lWp6t=G|`^Z!>w{f$aUzMGd(WRXs)g)kB%xA^zR~T40ZioZJk6?X%x* zwRoPJy`iU_ix+jDWedezS^@CQe4UON_bVQ`8QoJ@k)NrO1$o_HBPsA{z1|9_vE9EW zIjy>N?%8t@eP=U>it_#8g~!gg7^%CC#S4zAgzJbUe@3&id5x3MyW6$o$ZcpEykY3p zN*cuf)H_4G08A$pSM$c`0-kfT{3It}OVr2@m|m;+B6DK1H0fHlhRYK;{4GDDGSecs zzAa-k{=TB5P=vU^VKdG0m=3EuGOf3v(-Jgx-&B#29rRK5cV5I!lEAW1UiYVfVb;W5 zs@yL`lswA_e-!V-r{Er}O$R1&EhnH>qe+wcRj|<{;=&nRsS$EBwuUTpa#UscMVYo- zGnB2$<*I?usESx@4Xg8lp!%hIiQ%CBCoDB=HaznTR|}_cjk9>wB_d7r;3RO>(iY~? zVpsEwob8n!8g2(`bC60}FJk3`5he!I7O2c6Tw3?+#*uhG8ib6-D!)^mhw4RUtHf){ z6Y1QneWe-3<*;3EJ31NUAVo#ZC4iTF`0tM{Alho0K#)HWTIaHMmWjl@S>>Fv8cFHs z$o>hO_|vBzkkQ*ck9MrHP0Cs`O+cHEjxLfF(N&A8rF8>pr-Hlf+JnJPSdaGzJv#wW z+6Fm%yyTo_PnHoP;f;f-Y8-6+k{PbbKuv`pB$`~b%or<|Z>{$F9aMIKi#uorhEz$g z9jH%d!KjWvYhxwU<;*v@+W&%_ z>F7V2{u{6r^a;QhyAiaq*|_V;Stp!#Bm5%hY^XA-zDha~X3k};@YqQ$t9rHJl_A1f zB7U6B)T?XEb@u$VU#>g@Lb0p$I#Z!JD$X2XIs4K)j+VkWD_znaY-?2vFhwHD)I=X2 zU}o20PD%t`P}h3rjO55A=T52t@szTFy{}h~imSut$Wk;ye!b0K^}*2MQJxma(E^Oq zVIa`}Y1{djl1r>pAofQNyP-|>+0ysN7lIjlm0XsGzwye#s^pEz%l^>L6+;r_Up;1084g?AFjKpA z>p45>dVl#_)mDQZ)cg?4>L+7|(lj+$<=7M^JvhJU3aGR$RJ5q#Tw+;8ExRLWdC)gF zQx6(Chz|Pb5k$-9z+5do(CUKg`*q^Yyw4*o_Ve;&6wC9^Z*H^9&6@-XTujRhY%Snj8pd6JWA|~@#HFW zq67!Dr66V8V>FjsP_kG|nOnne-{4G&GKyodL*tU;ropt7o3V+5NXQ+{Zk>aYYIBOs z_>`#43wuGTc1Tqd*NJpn)#6g1`M78y5#a|IsTvKz?jti*ztw_z{J6NnDIQnXLKI(t zgYHtI#>`d!mY^aMvRHPy_8uXVVsQo_V?U0q=?{QbLYj)tfrN;N2p~t;XusCe(C~O) zPqM-{apILJ)u_ibtjkleyuzMl(5nm(J*r{o_|dmgMl;--Cmsb!Kvyz>h76U%WDHqpgB{fd| zcmMRTf5;|nkUhhGB7xjES`?tE{PcAC-1-Er)+1Q~NMeIRhr(Jev0l70f@xjy?V_b? zG))?lhlf#Vk&8HWrIXGZ=(@ zvoDfJ`GTIuGJ8h^o1Fgr?1q(Bd3dYljUHDN7RiV9|BID9WlFKV=ZHyEau6GlfvOG&Zq zJURY$-Iq20go$CFD#Is4rFo~)S;(a`oekv-}#hCYO)h^AC!e*P=mV9@W30+nv9B_uR z+mg=MVC`8GUhG7p$wx-m1XQ=V= zIMr%gEb+^-WM>uFweu*AQd5pLa|pXzP)Ed=ys=FA8}UBKO@Sj4&+YnGn01!BK@E+6 zs~WNOen+`*2X<^|CMgW!19Lk{RXI9eK$2{xs5|)s#2*8L`@ngU&eE|(%W2UP)ag^`L&N-xPFJxu zE5y@&wFb3ged^8m*kcVLQ&7ea@9KnlyzWPgLEFunb--$QvMDVwGgZLg;^H+eeRJyZx{}kMm;exO}dkqMx6M%=YU)OEy&O0;8&v z`9k4?boyo!(-}rKSt+UE7(n&}>e<2Rn7>dgPB{!r>JAL;?0Q$>Zh)VAhIX|}3UPSs zsDRW0X0DXPV-2k*W@Q`;J7KM2xk{GScC|Jadz zrbU12gyqt^G3>#DY2&C|lPa$N=TWZSU*o_-xKIFnd+^(phZ4Hru#gcP>Zyz~l1$n$ zvQ~#eRhn2VT}Xcsqg@6YoG_INMoO&^#Tr!AMGK^+A|P@Ggz1hhKRE^EH;5=lX3wX3 zB6Olk-DHr@K5XVbuT~c_&rdR!9sGnvUzB$CF)tDB`h4;5n~rOC23OjQsCpxge3d6O zb5ux6cgj2;r>+2EMH`4|ellAgp{ z9A&GuBpyXk*0OwoGub)L#i?S*&D63vnIse1mxr!=LYuHs!DY|DOhsWh%k-sFQwBiu zf3Ux)67^fG0Dd|n@#lgD?_4IUB2z%>o#`{a$=qStzCn{|JX}vZ^ zJ5v*L)Qjm>gs8E~%2eY4l+$VZMDK5%)eEzFvoKKLp?OmVU?P))dcMZrL(6xBtO=v4 z{HdU&MhmnnPnm*b8&MOEtuGi)|=vg?J zh$^COkI}>wK=kQ1nwUGX7Vp3yJ<1E!Sjx;MI;#+UhtQM}9zg-sr3zaWwc;=7;Zjnp zs{0dglvm%8Fwa5eTE0-P6?9HZbeKwj&xQvJx~+;Ng%{dzMLVElVFuXaForJI3ILwf zpMMX#!F_LtFa{SdTa_E&V*T{8|Pg3i`YqdOaTM2CE%kyS~?f#p2T%Ih^&k zHWBfePHU-I0OS93t4$M_K`rW<5pcIQ0HRgXz#ntl0IRz}$E>yAYi}!M=OYKmvy^sN zwHa?uRrEivg+|0O^s9KCFFXJC($N|B?EmuI_IcQD^bq&Mt8EuFS|+C!bnI zzla^xrGT_i4;ss_H0vHk1H&5H0Kav{fpn3_xEPVE^(DYSR~&pciV_ssk?b0~4WCJZ zY-CX=(M6-I)k^HJ;h=>c2{H3fq~yZ3N01vyV-fHp5m_Bel-9>1(Scp2h}$Z(fa;)9 zR?%^@lY&peJ@W~QpzqVkvY&v}VhD5cCU#w0RZaOZ)y3E-Xk6ejase;bkbK1EW$(0Q zOY9S`+x6{AX*n~pSt%DXv?W-22hQ@*wq&1rL79h>1O*fjhvp>=%3t$ujG6Kf*&i5- z#f!d3+5a8nsQ(|6d4{7-l+g=_memUHyq(BwEr|^V0)?##mdVbk2XT=Y{sjh?nnar@ zS*gFXLFB!@(FgHjr9Jnf>Mt(;LKa2IHsoMJdfN%GZ_HKpXQG+70k#upVoL#P{^Cue z%B@5+${Un7tSEz_{p3s%1zXYQ_Gwb7kbZ1;bJ4$h-~Ny(*^h`&aBn0*97T$EwXoeY?K9DfmOt+C&=42Mt=S!qSF7y) z2#=k*@t|GH0t?l4Gz^GoF6x9^CbDeBAT^q%BW@)mT~)=?Kwf=4ZRov!T$rKSz-*Ct zS)S5NOTb%6h?g;knn+{#-1wt3Ipq-3T3UMeuc>56ae7+j{yu@&DsLO=>_`^!<>d$J zVcV4u>0pU=?g?6ofYk3+9By&Z_%ic?*qrJK8>T+EQQ;_~ZO84cp-NPDdXTrgIEd zOplN!2Hy-Kf-s5TE{7rJ;XQ1%T4dOaz6`K7M(<4H7|AsQPTMrbwYNC(_My_wH^IQZU{;>yO5!wpo| zd{~B%8`aSs>hd~FKSc_oq9_S>1eXTeOeV#j!!WE)M(F1j=^V^Qj6s;R3r+tv#7?~K zW~35;N9Iyuk(Z3WdYz%-_5p_3bAhp~#u{PhbuM~d*2pU+B;=awPsrIsXvqnF@q z(dtyXaZE@KpOCOC%=3On>%Hl;zU%z&?vTIKl5XqWXOuU_GVj=!IA8%=lYJa0uMK}YGk%@|o|b-X zaV_^W<3b#As3k%Zs0d;x0#^8^OQ__ecH?@}Um7S{{Kn*$Pz2z&ooonG?KgGMA;QK* zdG2|HT+Z6-);4qyaareDU0|b%)!~_GX-7x#xx0AEDBP3lpbsGPCZGa<3efpw2=iM`5$e ze-&FKmBBuJN~acA%EEx1gw4Q>$z@8s1*^#oV)$+}1(kLa#0PR)0DcEj=-nb@E|d`# zxrO|iB4?<8|5I3|YREk0U{n({Z+bKZI{Rh*0Tw0)trhK%j%qGsw|19h&5yfyCv#l8 z+<1qqI5%zU;RMWNmYS0q90%ShMJ9s;M(-dmJi+KjSw1vA*e;$)pczHrwMRl;9C2s2 z#F@bCO(0rN#cC634|YvWf1VTVu9t7Ph+sUT*kmHVU?c?1fAA4%*47Z z(?9sC3!RA7MA>)IRwp)~qZs59ylL(Q*XZ*tbJS)KqHCKiU8b({1edUlxa+&nxSW463=Z9+O@Ti(9YmhdJW~Rg zWZ=XqY6?P9a=;lY<*&!XN-a$x$~kPKYoXkBpOgr!Zj* zxzhzX<(6cMTVGUNdZr56SF-MqX;atG7BZo z&tm5+{O<6qcHSj#JStKs)G2Kv0OpaSp3@!PLN};Qi47jIJS9)KTb|6|!W~M?$-QR* zLxejaSEse`B(tlsFDu$gyk>#v0QZNuqO!a8hKQl;i+SZ4{n$p#?IHO0))L3 zqzSb1Ofk*ibp)XT%|d0JMQm=-A%~!plGHcsU<=P+he+K7dv_VAI2vhlTO(#_p}c8> zoY>Z1AQ+VX%BGS(9eUN3QG?4W1W{thPfD}e^-vQm!Owue1RvX4%#P;`~XW$qEw|nZfO4) z92_@mje21tPL+r|%W;hw3OQmT3F7x=t!zm00=eL5DbEPf-G@cF`5Y`p_IF&k&U1Az zk>||z@O9$DIzO=+=4cg!Do~a?V;&bVVZ>s@;hLybLFsV;Vbaj9JF-M%>0DEDJVAgfnXSsf4*Z8 zswqUt7@Gx$`7<`|DKnWIU1X}#3Z~jf)Mi)V;Vw}>Qmfi9B!xv&ybg#&K zkj4DSx1w_qBNI}83y#c*Y&mTwEfSNxP+L0}JNdDmRV>fsdpn1!puocrXCfV};!tGF zK_0d~xqNCt3AxB(wcHu0qO{!p5KP8B>PGzE4DUOqEAYqPnN~eNTR8MHGN1KkLvxv? ztWXwpC_8PAt@`)Vu4*Iyfw?wKj-jefLa%ok zsgo;Z{ZVsL@NtcpBj1lzBjrN-yGBLM+4L)r5vh)`;85@w%hi?wjEYdPBy`CKc9iJa zRLy7})u|}ZrV-@$BNK6A_$fs_;^yr*YM+uK(w;&=;-zq@7)M6bi;0=xcPpvI22s$i zLk5U#a`c_p9g?8BTxHp)=#}Rit`O6JK6xpC8insEbVffpk*LSgPh$V$Sa#SgXrBy( zjTfrfEzE&5%9+1TLx89g3l+QXGXdQfkK1`f8gTKUrhYWz<+Sg=s?W=zrJ>R3a6re$ zPq;|VIq}Z!SQ7N4=z>2Y`1ba;pdtl120NdtqDgl}8AIz9*g zZcA|Mnv+1Sap$IKe;}!U1JU0}!E82<<#kD({3_4>M29bpi z%1Q^lf24e7uZn7ORjc^JhjS|-!lYzJhl1F4w%)=^O=*kvKmma$@Viv|B|A&T&VL6i z1p!+NG3G)G*AoJK%%*%w0g{*ybu5Wat^88iIvCOqQzHApUf`x_Hpa+=VL0XnN0I5& z!;b>Zp{Xc56^epGjm3SFtr$Wgk?lgLB#Ej9Lp+{qKC66p4n$)|Jq21m-(8wGS+4Wfa35FMPmK0+T1pCZ5W9R>WT&mh?KG5AJt!SIpHQRi zz!JUkJ1Eeh6Ho@%VqAndFm1HT*cQN#lAc5X2hQdol9)5zU+$;I$z+QTbV@J6V&6{}?2Gm+|-s*ZQLs&=`a!_&llFtrSe zjUQ&q=?0vQhag!|!q4!R93&ud1Moy)MuwRoSLoFzP;7A7ZM&Qj9|X+w*(ppen7JzT z%Cq2*S(~hMSj|ZfHmc`gaKYTohb)V=J3vqkQ#NL51YUm6ZGA<_frls-#5O=Q8Hi8r9mNd6FD~kk^7|Yn*8Uy{d+ZZ~(Z}|5u_E2C@`JhcrggH>ltv6qpMO}8x$srg&wOb@&Js`IK-^E8TL8O*=#{+EE1}wbq0+DVLwpF?gyPl4r znAII#eyNGLY32N0fs_ml+p56GqV<($h+ogAVbcN|^>6lkr%V7Ovsepivvq zLknz>&h)3qg+_|A8QNx!M4Wg6v5#81T%ly(f_upnpE`eyw7vozEz_mYnPgBzIPOqa zur$ylUydfr)PWpjpx{Q>_JA}e!qZwG1Y`ZpXi9@zbVn4Bt8*bcomUyaZ4Ce24)z)7 zdLS8@n3pNlb|^9>BA~a`M5$BKNjSX1yZC0Vmakuw^59V1WL22fn8R?~%Z`ZD3ps%6 zaG}l`ZDB=fmYFIGH=Uy*z@9WSO1wS_V0ttU6SqKuMCi{-^CRCwl;v69e-k$p<9ywq zMraxIl9~e#=?|)3Ds7}fM(sRap)n+jyy{P8$AT!2UukU6k(;T@j8*+{z(J1JTA zA}*Z}<8brhhM6*~u=5`b1w64D7Mx~kw8IFUexD#^96BpQ>SH+GHYY(Sk7Zh zm8kWhdt2bOOEs&Th@S{gwn`T=#2F`-Lt!Vf)yrbTOJTG~tF7NT<-9YAT4d@gq}pY= zn4}>Ba$+n7WDqP|6jns3kiX1#H&?WHh*qpv^VX>cco{Vl&RDx%R(oPaT3raSGQ_`Y z!Dt-z%A=BXc9D^Peh{=o zi17K+0wVv-ipsj*y)oTe-=^jSi9k!)k4zFbv}n}of{E3?$FZ2b<%y^wX}HeNxk&%xF)9IXJ!Vg_S+N21UYiX%ol`ca5k(X2Nr0+K9V z_!;l;^Za<=bh>37QiB@`ka*^rQHE$lYvI(>Cl9E_(qT@HYV~nj7r@)5|HCsHD$;xK zB_Sl$WDgh^)nTQVwzdbW3@Af*3l~bJU&5_wwgqij@5Xd*iFFgDEMIV9Zu@uFAAq8x zAnq6^>vlPrAK|YLK~4lR>Y{;mYW+Q!Tgx9bDheB`*+}RiZIJxCFf$G}W})f*BPKzD zV(=;3{1#=gD;rd`Tzfp=&jC)3OD+e3hnySPkjKssn(w9z{q5x3UWHwO|B^dZ;M7Oc z^kF-yKA((2BkJVRG`R+;t2HTpYO5s+QT$A#n!i~~SE>T!D4cI)KzHEbLF3@M`+Hga zSDGGx%mzj^1WaBS`)RZSye?iueYQW|2=oAj?8KCV>(Nvh*^iYBa{_@*=!|CD4gu}D zdo%L^(2>v2g23BNB0Zt$`}xmqe@BX?G0HjLk$OVsm)m92?- zQ*3r#@ElW^Pzv^*_~Aj;)e^_E82_^t#)UNHbWe?J)J49GWna1@^dWsmeQr6aN?CgR;bWp@`XTg zM}^=afaZYb_kf)|4Xgc9=_E8#IidEy$4WbG(0MHC_1>+)HEV2N1#!K~G3LA?$D0&9 zE(qr;B}KpeGEXdm-$>#V2J-5Rw&47vx%_@C3(1(xs!hgLqr*$wP>5P-ik@v|5o9F# zi5llB#xxC8HxIIoMTtXnqNu(lxOtVY8azPQNJwv_+FybCOqTOkI7Smw^(Ituc7D7f z?)pq%AX%WBriwED=JAlS?biMWEFGOt+FR79IEqJxa8@3Rd;|rg zp+gqKNC3%vOBJ2$jNi;9TO)9CW(1h^t4FNnGoGtjgN3JsBaqR#eq(J_ph&pWW~(Bk zDVZ9JV9VWe2A?xfx|J+u(WT1pS1H37>AUJ!Nu`eeC3w?`#8@ujcCmk-B4 zq{zOXR%tW|><^1r*`Wl0i=Q0xRvI7F$)b@TPwtn?1i3^?zp<@0dHA4!_opBISjR~zy_BM! z`^GkDzdH??wgz3@A11q^KR@NfM&DigSm-Ks=$f=YZoR@?i8Ez=A7mRuF>JwxQC9v* z+0t!+I%j9omSo8Whi5#mq+6Q0*zp zRLQf@rU_BFYe*m5l#xd_XGnavf@o<*%*HsI*qvfqG~ml7#Od zcWh#`u|Gd~qkqC=!FG5wxAPD$G;|L*OXuKztdz$e-8e%aovUHWtRN5SdMM<@+`pS5 z|87MqGT%?JSXOYkoL`HV(WYKzP-lFx!KI>(cPV?Jc`bYKJgpqVZ+D5Y;h5$bP05O7 zsHjsR@`n#WPYA2E-%XmMDK287Xpvh)WVDt9Z$LmsUo(rXI-umww5QMywQ|McZwXN& zcDpoj-@EfL<1z?V{*CzM!W;~=b@_3}Da+GV7)FpOd$}f+if1t7jZ;#t1vfMRB)sGO zv5*MF?Bz;r_^R6;@W8V{pEiO9`@g4f zYx2KoyMe);@vjIAxR?_#9B+xi_^x<=Uj<`ICTve-(XP-yLT|DMX4Xchv&>Ey zH*~q*9Y9;4S?5HLk(wG4pW_sNRk{j5qprnZ!!e=O(0^+?am!8HrfhRKFO$VC+divH zvupxwDy{ml>qld^gJLH^az%y}TJpF7$Fvv`A?$CX-ze2lotAO8yO&r2z)8V^kLGcy#vMmU7VBj@872V}O54Olw_DY&#h84p^=hxV#zikcmPVyM z0-a>Dl7da+S@5qct=h(J6JsXO4IiwUm~+4`NL2EBgwp^*?pc3W?nQ>Q15!2_G<#U| z6fT>K;!>J|?3f%Tf+#0Fq=i%=JZ)prvw*SJ@jK<7K&TRd7VL9Xs!F_Xx?bY+?=Au- zoD2r6ptL=x4EA`}B2CDQNw3(di@w(yY;2C{8sSyQ>~`Puc+x z5I{x3cfpADdMhUl*4;hGqv85z_wD`B_R#LNnc{ncsh^*p^DQQDCj@RY*xEC$=wO*t zGVYobaWsh5c(;Xa)F=eDCW~;aI&6*D;%PWw-FRZ;;Ys(|)|MU4ScwL9>98fg?q6ruK(l;p=#C%V|6 zOnb>-kxY#ZP)})G{(8n@zWwJQYHLV`4*PiT@{{D|2_F?>+>(p)}G5x-b#j@WR)I`x(kXj9VCuB zRxm?x2_6}pqbK3~B?gC`qG@0wh8K`%A`!0FZ1P0M=Wt=BG!*!EJ8hq1{)u+p$J1)r z*?HUB0jjLeJNSXO^Hs^X^?YSyf6g)&8*Lna$YmHnsrg;8ufe+oi`T=>;C;b&PGwS& zcKXHEkwh(TdTOLAyi3h_o91Y4Vj;!%k>KIt>+_^$t={MZSi1QIs2m4wW*QXo_^?*~ z)q$&fc%Q!R2t-A9;&s0pou3X#2HtEf4RCoLlgC@`m-_a&UB4B4JvU5GIi6m8+xDV! zDQ*x^HyfeEtnLCV$>qQN_&7M~TWcRQzdoKR)*8fFFDf6^TlM}8{rUv>Zh&`=Kv?$8 zpmp=1&NgbWlBwjq>S^{w_Lcjy(mx0UG zD{J7tK;^ifeb$HTJb_>5yuQiNPNnV1TgrbLr5Qzo=n8-EReh`_kBL}+o>7{5pgdk8LIz@Z zq8YH@QndT+bI6_saB%^Mvhap#wIbegTh>nSqj*mx{eJauSp2mPZc?Li<1c-5boeQ7 zuH40U2qiesOI0j|t~4JH%x6<$H}n^F0FtCum)(~CvD9ET+fI+aZ+d`qxm~py{US@* zR4C^UH}h-eEtPd!0Q}$`q!|w-_!U)%sWKb9MftC4xp5&fxY87LTaON$FRS~Reu~9% z&6?BEiEIP{k*e3rV)x4Qw$BIKoj(FCrZ+g{jH#zE`xN!Dkey+0i4LW7R`M5d+uslv zwE9wPA(N-EXvLh04nEH(wj>EvnP6xNR;fC)2{xOIR+z;yddw8wX&S+7I&RO#%*=&@ zioY98FI&G;HiXTRaGS}()F3JoAsmvTsmwwzmJ86<$}7@c0kjt;<`JX7{e|lHdopLk z6o|Yx6LPx3p3GErY0*cOqi|GlGWJGAdOa*vOO$n#dUnEf*DFzI`sMvX?^ z&eyr2AL=bNMCTbWlt6F3L#?bRmDOC{p%uK~bUG{~IS_a=0|n;j!14YAMz zeLE13K&hzaL+mm($@v89P%2dG;CGYV6p8D9KbUpY{y8q58d99roP~!41u5MI(7s|) z`q^y5HY*$V!zhf(x@SR_cR=ymp2vh9#raenqHbsqu+ABi=|_rw|KTuHyq0YNx(ZB* zgu*IuF+7h~%%Nzuakw~_YLAtrcv67d=QjpHUKJKzH$9mg0$Ww6QWBzN1ja4*R~sg)rXF;YLzIpElz;X7 z$=j*At(GycTb%dz)B1c(Y&?4hjn5gE+fzMYeNf0B`LC@yF&>fJRfDP{6pu#^3f_E# zaLb2Z4dj3E1Dj>yJr*FBvEYe4)9-Z)29oGw?W$T!1@k8H`!%E)L_>qCL;Dc#>5`<0 z5=+j`ybG@oEOEwRtQwQpqjW%xl*9h#qK|D_T1v{=XG!?|TAg-3;70}xl>o{(-HrG~ zG`N2_v23upu9Nsg_^DOmmSRCLP}`ptVOfQy+{W{CMss9;VYaiE$@Qe0*Wx)h0NWP> zpyQ0)Mu!cnq*x+#?}j*EX0S$K4iPGu5s5zJ`7xEyf@HtpVY9(z|8o^_oQs6&dn(wY zJNtm2i8-`=oyu}rW@!D1RlF9-L9stbDPxP10O zt*70Nk%g+4kf*b}YEm2;9*m4aeUe^D#5cAK+i9VDYwj!vXqbbNN!5B6O%lDg;|!qZ z0Kk2LJ3bn#?*9(B1DbSH3w8Ah6?BPDL{MlCrdypJPC$rN_#h!DEg4XHP*dB8>7h~I zj9ya=*rF}Pp6M#M+oZz#BuVL-{cV2#Kbp=mEUG_R+b9CkpaRm;B_Pr*AxL+3cMhG> zozmUXT|-KDcXxLTF~q$4f6ja68y6plnZ5UKJ!`G|_TNaZ1--{rjqP-k&f-N0=q2Bt zKtFkZ2rk-ggyvzjW-i{CwyEoU`lnCw2$i50V3#!zlQrwan-Hj9j6#7wWO3c@dL%(V zlXCMn)eAfZznR9Xap`WOp{gNL*2}1`dbFB`6;AX>{5P&Ol{CD+M<*wL0$d7yzwnOK zDk%CS->Jie@PoIRZ6|HjlCsQ~O-nq~g^zvXz7kL@u}|@yaW*qrbmkZc^N;R}|xjiuLv7qo2p5{7(4K677v86fftIzz|J3f&j@W4VV)#l@b2- zUcywrr#nWUqx?u+0{)8hy*{h|?g^DCr1wlc3U@HY?|AeVL?2+kQ{g;v$P=iS9W+I1 z^g?49{MNFd@&se=JBc{d6bA^<4>aMdi2i=ab1L!^p{n4?y^f+=y}QS828nx=GWnhd z8S6QZ7Q*0n-QPIq9)BjEgNHDMG9G4E@lX>aZSv5SFtTdWP*V5xQyH7=IUg=E7TS7W zr1kdpuC`$SKxwPV>af4u?RV+ya#mSMLgevW@JUnjtv&&Wd14hpnF*hsY01b~IH(&{ zOgXljAKqzP{LHZrWMr)+;Is{H23Gt5*z4|ogWRuQUurlmv#F3egHbO&<$Hhj9=FUS z#3MY=TIs-q3;vlE7YEo+m8|o8dir%*@5I8n8gxa7RIH_mVvB8W>XQKPphsD9O3Lc@ zI`+yB(K$f%4aj!}$1Oq_;B(e#eh?87uJE;+08BougZdJcx+FH^Km_Do|92by?{I^` z7wvFN%b`=QXB(|IgSppRae10Jdt1{x%6~ z=bKiH&V$*l=I>n0C3P~4TnrVzqp%MDIH+tl&Uni!AyctY0;<}NUhI9p@I&I$s~}Z_ z&jC}r1Kd~Y`3;$Wh42-|Lz$koE71#8dc0n4oddWHg`a`V3}O6}S+MvEH0ouI%*#4v zdzQ!LC@wXX5D$;0Fb}{mq-ua^`$dI`(&oQ2@KZz#Bkl{VxSl4Zu2GZR(kn67*>Z1y+1G0tfogFQc{q5|-0epkn zIu+iX?FE2vJQ*dEUSPuY7QVr90hq-QS9Q@MNACEUvQD1wuE`+(+IU03!>DOY&jbQ0 z;|O5N35{R7RTgEuSf+Kn!RqheaQ&?+z6SUU<&4%4z#iM_wVIPcfJIn)6+KQ!7bUCbOl4_E)oZ7S(YHi*=dA&8_X)Q-k#!519Zqtx;PmvY7(0 ziRm5{&B87v)4onL;c&WRCQnI)W|PJ8*Gxy&Lfrr+ou~TY;RvZ;SPJw9I|YH185Wrd z9)mJ%?P}RwVG`_n=Xd?T(YM)Y?V3wiwVH;k*KmIdKTL00{%zi8)O0?X>eSFgr2hdH zDnttPkIkHE^d2Aqc2SeOjE9HsA46g0)07p7Js8HSmH;}o7L<`aR<@DDg7JlkDa(;7?I&eh| zelt0WyHU=>UatB0;c0T`i>TcR7uF7Z4sK?o$H(l!K3uCLI;j*1G#*4(`VR~z*l)ex z)NsPpufl}_Yn^5I62qfNV_AQv3=*64d8o0?vNP+&yG=3D*r~Mk$vvo~v5s92KZo*{ zK(|FBn@>>_sNf?-=kTK-tVe!tmGIw+fe>?Q-*dF8$@cwG42PT|46$K_?NL$)KP95} zyp=*=+Y#m$3MvU1bc&fyru9!$5viUM=z{am6D}=$JX~;#mljRJjx*`yo~jAV_>n4T z^rpO#v`=___cPiI`fu4eJc&w!DJgk8?9+4);p2qpX$ToD@h=XP1xmfGA8lE2zbdEI z^!sgu<{s1|qSUL;qk;|N=)w?LjriWO8X*mx$wEqlgEoOEx=`MGFiQPnfWCG3ZXVaw zDh{sE9mHOFQB8q*UzI3DO}+h}{GNwx2_Boeh||xUER`A(@>vloNYnS3K?dX`4~D;a z-EWXtD8;wrOP-3n_SBqf@JNac?9EVOw$pg$T%8|Haf_a#=oPR8ki(eOQXQ{%g$qbl z8cw*JdLOv!^ayxuQN&2P+}9hQ1Mc^=>Du@pSs|8&w*MFj=e=tlcJA9g`rbCTvJBsi zlD;AZUL~fe?{pw#uhtuk4MeOf7@nVpt*+hy68dZUOM(Jl=nG7{%}s!&>E*BnFccic z04}?}xK-90K-h#y#8r)SgLGy44n3sC1#*ouBA_texKRArUD|pyp#;;=9v^;h9a5&o zr~`hUiG>7uo_nITHp|<0xOR5Az0i;{(gMmqt(bLy3=BQ1dTlR0tmQ-Qmq`gn4lv7l zMc4+06xU&!u1Q~Y7vCzOgU{N+KIwq@fSP(DgZqemty-Ids4kqbQUPQ?e5MSu4y(h)(T z-2`!#8WzVJ1?J~KKWa6ni&5mgF~Ra0tVo)^t$O%PFi4F7hI`mP1ftp&;VC6sjyP5rgd*j z?}-sVZr;3dud)9^KHtrs(^vVxpxb!*O|AI{%lXoq!5zV6u+0u~GOELsgVKfjSDLE! z&fMW)wF>RX>;0%wKpTn44-J?QaL4>pe#FQU_2%&rNvcw>@PmN$txiSV%BFZ~#myz= zA79voo=;@^PIQ*Q&DA{jSzA&~l?c*Z8~cgIY8}Z&NOW4Lg^u>IC#f4sNQ>t}Qoxfd z;rG?L`y7t5*4O*amrokO?1S4#q)$SIhSL+UoeN3HN-zg)`qjTb$tdHU*V}ef$-K@- z-uJg5UO~N&{@^N`wKl}6-g+4T;sLu%_Z9TLOGxvJsyMusWX4VOH>UyjrmwoqL?9W2c%!9;z$HQN}&2V?C#LN|g;oA_)+1sFsU9llv z5a5{dOuyar%OFm10$Uviq|aCJd6_dCiIy}!yo&p777_uvCqLh(`?Cy4{0)Or3f#XK z0uF2Xz+7tx9sstQ;#~(%<~aNj&w4oeA2rp{DtXV491*|AZ^w@oD|b5mYtB`!PZV;m zpN{N&ujCFUvhOTE#K&hrw>FFryPpn*{ybLkJ&l|!REb==I?674@^XE>XV@*#>i8M0omm}cY3)GKG6Ubk+OJ&u$ji=;?@ zv9xNCYN?0${as91(TR24otPr~P>EBFns-C~q)}qj-|3n)A!5QcF1r=k7})_;fem_; z(+c?(Y6C}{MngRl?^_C3d_rZLK-M5pWIUp)==%!U;%6|9)%Fgn!?Rw=uC2%GV|J?^wRnEyj_-SCJ&muP}#x6C9e>%#h05W^)yKak2aP(7yFI|QOhZL= zf68jVnG>gmf(j_$4lI4`%?>Kl)d4(rDzJso$?7@}2K}wlVVpN3&#vDt=+`41Wg=Sa zJX!kdi_ivOt4RRN20pLBvD5^tUuse=^;KWk>lygs?sn)SgkYAG-SqnK%JqPh->qZw zt8@N)Q(e^!u{~Cqal0u@lJAx~Er~$PUQlS(tDnQq;WE$|wdQ$8$i82>pzD+7{(WEZ zOMi6;0O&2Ym~nbfHzDv6F-=JIX-Y^(lAAVG?6VBpeB>{y$)~(-W+teoDTZ&&_O&P#IyNkFklkcQ1pRL#%5!_c3M zW%*OI?#l<;?2ABHX)NyvlzhU_NzUXyIv0~DFw_92!~>U792z$dZ_+V}_z_o;uoU`Ok18Z?iB zZ@u@s5P;oI0GLnm7(6*lr=w*vfCCATW1v$=AW{CI^taE@1K1=zs~K<(@NF|^61y46 zT+bD&;KAg(qey50Sc-d-U2C*x03w!@%XoCos;^A#YtljA@2-57f#>Cj&aN))d2pD? zO|VLen%m`7_H3!f$q@iup8tfW6Zho(47@c8DK}8o6z$U`vtNMvuwOc*M*n<77p>4L z<`tkct~OeX9jj!Majv)?3OJr70BBYqzl{)G8EijI2n>7xPBu;UU4#IfrY#!3ws_<> zS^oX`G8Awsp@yp-*m^spef-P!FnaOIpXNEEX}ku4xqcJ>4y#Cx#dCv0C*-$S9;MNB ze_bd>x_m3Gx(2(vPiK>Py3KzC#wNRc^RWKIcZ%`4Y3F5Gq^83E*U?HoCXWaOYX#u1 z51}%tEZSOPAFtdIg}xZ&W<@v1n@}kZ52h9xVas7B>^^mgK}6x?>Dx$h5-d`BZTV3D zd_+x^deRWpu{g!)T_GbtkW3Zh?la&3oA$8wWZ|`&a|zQfYuSGLjMZ(8n6@M|ifzbg z34hIpSLKZknrs&~n3Wn16|MPp3?a-J_MD~-$T6d**}Cuh^^|m2(G3i2&uc#X&Q-oZ zWh*z#JTIE1Me50!DLJ4^NmweO9^z5qe@ok!1AR+AS9BQaQ~1Va`d{NG z=IyaG9rN$8sLILYrIjD#eP@-AlhmkREfJm&IbJ1KT|2rdCS|!O>*RQ)!}@)y`$^72Vf=}4GUgd{?Kn)-e$OI`!v`=$h*yV^6vLD1wD_SA3;)r zI&QD4_}b{wZoi*0Ej~A`HM!moeH`FgZL@!U_II}fEO&+1;f`M#@yuP+n|~HRe-tnu zG>_z{(!RofUD2vf$H7s*IU2Yy=*YEcjV2n@_|^74<_zf9^yEArDGih-wAz{eB2)VO zg+t(cF8TF0i-R^5+JF)@Xvn7sxkatjd!@gjObrmcg$@<`yy%Py7#bYC+xKVDJ2l*C zQ5jV7x!J!m5;tpqz6~KA1)`biWq*<27zSpB%%}MXz^Ex)vs&l5vf89e-nqA#x#UPo z6mSFxT-Te-4odzETxR2;jE`mC9BbL}#c2PV2LPA?dL5rZp8+R)uM-C17#_2xyn@0x z+QgpSebx&L(`j2|t@lhKL(o7paTb6nKJgirad%P&gf*?ZGxvZ&7c6g&XypOHp9F%s zw$Dlkv`qkvC;bKrTHFOL{5I^n(=}4&l&2(+kiyhne#l)X1X?$a5Fn0vly~mW3SM?T z-h?r0(@yRVFft^2*Ltr6GpI3Lth5rds;K|TiaQ#TzSo5Z`mVRKx4>c4Q zV(o5Qy5 z8=3b!9N)cmb(A^2b#Wg=9bl3y>e!!4Yq+P9;Q)zOUkw>817qKqPoH;9&QCE@;FJu= zUK_oJLTKY@lSYz#nxQunz^QfAJdu$m8zBq>rcCI68RU z$guydntfKS0<`R&WhTZzlMxjrN0-tp0>H9Q;{rS#gg7c-hC5K^YP?aiP4N`MelLR+ z3>~4~tTgtCA9UR+Z$;n(!dR-fJyzLPML`=LTZdEdYD{{o^*(!f_v`eGjAKJbaNYCo zHntOv=LaanWcqdZo*MqLkSb|=ncPQ@0oeOhz+|)Ha`-lIp}F+G{wUj7F0+8%GnuJU z!f)CGJPSazzEFt^j_|E0p6rUAh2V#-Y-Kj-t+!Y}5=W9{cS*YfyRM!^D8x62v#~IR zxq%4y5Pje0`>DaGI<;LqjtyrV42&VSyniZeQ|Y}R<~FY{e&LULDV2E??0pdFqGVcruj z$7!og^{Ym`;ZaD6YnSe~x?0zJ)sb%~<3sl=|Fc{-w+A!3V1PU0DQ#HLj%l^Zw;`w~ zTrWCj`eYHIgo5dd>HV-1@)k!kLv8a-kZ=?VyE$E9ymEYfT>siy<;LT|X$^bhgup=- zgT%p3E!1Ru-P3w3jlHqy_lXG|2As#85(ImZ8DXqz(@;!g+{?XSCPdg=7je=WHS6X> zM?E)d-v07iRwu`C0<_A0fn_VIc%`wi=J|pk$yCy#b^#( z*xKvV2e{vm#*hzqbbEuMwpsGD6x!lHCLr=3NYSOi=2*cP$reARSqHXLJr$-|o8UiX zKrq>!q_0rI8>ruyqf`HLo)OL9$?&I>KtY0~MOEwArgf$J(AzlwHj;RcLbh)IV+#uf zX^pu6L`i3in#_wyyKRGg-k(Jm#P)GQGhV~8w_He$(QFD!DIZN{ci0=!QLoB&+ zS!S6eT>@eznOfnn6pjDsLG?g((2XY!4K9)(p6bx{Z*KWVh7=;z3f;@xw53DNz>T43iZ4B-+lp}x2Gz4YOQt?=PfPE6f|;1Gq(|V9ZQSK6G@ip!*e@P z{I=k#iiAwhE}T}EQ@5>0-}iD-k&KYRz$-s^L{jfjL@HkDj_lM({fT z1j)GW#w^QwXu1h^uSf{8S;RR*1F*w#lGjEkrJo>xk$ z*8H_v1)L9W_|8Wk)-gu7aDP0Pe!##@{gx)%W8M$`X}(++gVAZFk#VKTPC-&Kc0xf_ zmBa0FQl%{EVzEQV;&oq!kfTzM_xpV1hgNN`la@4J|4_uWF&y$AZ_ zliVJ$Jpq?mudwNQeoxoBlDk}Roeu<;DpS3$kn(&vmK5Qwuhru%X*2H_Ac^M)pu-owUFQ!U=8Z&XDB8VO${&PaUIx-nfU zn>(UbrXl!zzWrcO!3g$J^%pdh!}bV8{pxk&T!{H8?Y}J?!`+~Ej~c{=05iL#_#o>! zEZXTeN(}G6V|m?nf9QXBUgiGGp5^BbGRt;5*60b2GMqWrhAcwouW=%XTmRY+XssAo z`Q;BtyPc;FXuQ{wH>j|Ny?V3CaYZe>*UB=udDY^ukw@0I^0dwsegQpB|1`@24gK~L z0b>BS#6A#Hv*YGwF0jvSCYQo@Mw=AOx!;jTfb!;VE+y~{18a-BC@_eCkRT`#u3Pov zQg*3{%8%|_|I1(re0(0S+2zC$$Bp7-5#L5>=coG)@(|lR+9l{b&cf=$s1l~&&AqrZ z0t?q%VFlyAON0Cq!s7?t-ewb{5k$MubXLRJMIjNfa;-Z5jeeBa3Q&H!w{xklY7#x& zs7b`!D8fv=3d1^qpo62kyE{ngW|K^Wp{x{_(XAa^h_L%$GnMB~lj(9EYmm=zAW3L{ z(bI+EyHpz_&!x(w_th~@e@P1!5uISZPHm&51#-k$drb;FsM8Xq+>r*loJ{AgoBE?B z+OsMFkn&<^kNV~_xb7;-dhYDu&v<=SKY_K0iHfs~jJFQ4DDa}CWtNgPanQ1{muIE! zzB}&dacORz6=+MVfCCHuSW&NkO3zs$^53_5udeO(YJUECEO&4TYw~7I|90W<(c@Ki z3@!~6lmkxL6V0lw8Az3!5IU0>Xb+iD(dWB@rMY9S^rya`ajgOS&@iGQ z#WwmMKdSJ5ZTwcJV5VW1_NC5YAKu(A4W3l*jdD{-{g7X5tX&+wOMupuvAD{YZfL(Z zyu^F=<2W80Z!nROE^MD%H}BcK&2Ffvg9`kP#G9gYej+qvM3}~7vJWD|srCcG(gc(4 zt8;Vphc8A>`}hSMd7r(t-xa$Bb)(2K$|?|K;2Ugk(Nl z_+bg^tOm-P`B0?*u`$gC(s|r+1%V1F-#t8M;rF6Mv(=@ySS504TWKFm*`=I9-YR#h zgcf9Z^ZbZS|~>N;5qt17%3N=lvm;WL{~0CD%~zzqNeH38R##ICQR2M{rTcn0q6Z_h#ogQE9>}3i)s$(^VbEf)Pp1fC z(s4v`SPX2A%i0xdJS<)Yey*%;)K9m;*j$Q6>vhqRt7=9qM@I9(SZr%ZPz8NEH!QJa zCT64^QkLu2d13W>!VL1>L_aQy@%5VvO&sqM5#PmCju%tZ(~F?kBtZ3Yl6h%~)8ffv zQ%m=F4YJ*63}g3O^+=R&Sn7r|0^t?UHF+scyJ(Ha{hz3-y!WL< z(>be}zD7=a<&~q@3Wc36xa!NuB(EMM!`UxOucUY~E~;F(2~uuQPvuj<1sI`L4#0zH zs+}V$(m85X4GwSiR}bWZ(TTyf5ER{g>l#&#FJGO5qZ9xkD~5F(pcF^;*=K7(yxHb$=WHqAN3FWadXZGY;1#`18;=pEHL~6A7L$o z0%5k*!c&&-B`yA^;6~DLlZA{newN*XLSJ7Hkuh8TaI5i@)d zja-wT&0_!Boo-4d9X)@lrX-ogcoA8u$&l8&IV1k$WYHGqWK{v%H5?t)GwQm6)lmzN zp?bxl1#1h+Pf+y4$zJ2v8a+`nQQj(1C(3oEvM*Hc`aJO|?|=QvF#a>U)%b2q=t2FH z%qjlmhKcY0WdR=)MY=b4_zSjrpT)ODv3Jz=1jAHprx<>m6GP{U?FNI?s5VCR|8(++ zCRs*vven{zaeK)1OeY|my%`xEUixZdi&QVmFQix?9DFz=OYbeu@{63_bvCn_ZT|Cz z4pYcA=^C_R%4mSpW%0q`BXy?tIn_Y6{V{CqXcwK`OnQq^^=K%Ec-U6+`Rr}LU! ztgnTUV~Gr7XHp4)5-6%E7ZuHtX~Tqzzkc(-rzozKnU*%|ygfasYUkRz9Lg7q;eWoQ z+o2)h-bpd>eHfcaODC+rotlg-n;G7<(N`}ki?l1I4GpZ+1fNEedfD8jARG0|>SGEX zJdeZ(TFl$Q?!|$v$dmPogD1i3k~AirWG0Q~&8P*SIisfKy1B;brF^s-O{@ht(d8x= z1Hyb>5dF)smEfIg&eq!j3CcR=>z-?KLNf^m=UxZ;$L23hkL}#tItBH=hFX?aN3B&% z*cBLmmE}&7lf6-1UQ9Drac=lk;iax}Enz>e?l#!Aa6rU+Gh4Gm&8W*}H@&-RLDDFe zc!*|WKAiX{au4gM$<-|^h}`{NYtf^r^|8PfEAh?oaqwgyG}t34sPwqBf3@UGE~W`i zVtK;iG(j8(){iBLyh4kaBt{MU5DNW?4SQi`5G4Pq$Ai6u!7J@1Ene)`E@u+ge&88VS`u z*viGjtfa_G3mOGng`*i8a}B5~O>~H0&re3I(cD|du5BGWZ%w5NlN zikfFlJ+TZZ^J3~ksheca3uh~h)ul%(uK!f)L#L}?U+yjDREBh%8?dl}%N*WQFI)@bc(COM4p=LbkiGfe{}ZJ2Zi2doSo9;L z1k9WLVDXQK6B-gnZ=}S)V@dnq9-M?!mhmkBwW1LmDYZzdR>Yy85&DkXE1FH2gEGo( zj2i_jO)l1y;BHv@yS^;{2jS`4Il6tgyS8w;sb{nrlnL5jh3!22LQLgEpT1Y2KN?T| zTGWJkFF0Ry^4sRC-HtPAAb+ZxQ=s z`EkYg_a}Z96Zx4OWzxdGp&vuJ5`HN8Mm4M<3*}PMIyK-+%S97dVN&fRiMr#I_1coj z2h0@p_rG~TQcX~+4r4vPaXwjBdMQ>TTqyz2Rp^%tN_iONhp1{lD4(xTd$JVA6?e(QZ{8RwBu(7-? zl5yy(<~O^sQk>y6{_W5bJZ-NzW*=w|zo*L;%Rf|@k0~^AXPoP)JLLSS-E}Oxf)$op7-|D6LZID%aBqGkM$^5x8>+g%qq={2uG!sNK7v9hy@_2i8Prx&bkgP zE4B^nmG76cuyCKI5&)g_>A&!%1X4ls$w|twvY((`_N_XfcVj;)v|3vlU0{bSJ5QLO zf$+)DC&%C@o|7h0&~Rg&ADkS_s1vD4r2=7)Qlyp@a6h+V)^$I7LPR6**qzPCJlz-} z*(0=DY&2P0PCU&H2*J?%2E$1_xvssgTx;R1c4lmpEMQamOsJ~>t~ERYQVOkeCGvOn z%Pmf0?FWqa2nm$NC-YHqf{&Y85eMz}0oma5;F#y<$_tKN*{|+6NCo8fS)TjB&Sfzf zj>_i<+GAs1+UNWNex@3snp%Br|?vJ@x zvCeiXT840P>|$I(&N%h$u31Q;N%%d^HkxSmD&=#YPCD9K?@yNn+%Dx$-WFw?{#|Qz z`EmXd0&dCTJ4-Rk@=(|L=G^aK`$?z8*~q1qiK!SU&fN5GO&-D3XQ3lV_!;KREkBN; zg(fLW_ut4h93Gs^4j(9iqNzV66Wa~DNeYlAqut3eo<~SU$H-5ZKrMbl1wc-}JzL-g zYKkQ^sdU%OpjYIKtNY_9LLZ!R3ibJ~w;9!xMs z5#S}TUy+c*&(u2C>=2|CwRM2hErT&Ws990|i+`8Mi$v$)(ZGTbVG@ObU8&U06n)eX zQ-^V^xQ553BrUaX+_;|mNHcQ@7bkiAt+kPZ1(N|PWB5Vh(g=-B$NTg}FSSNe%tF45 znaJ8^0y=5j937i01&{Doat!Cmyw~-toHK}8>8)fo@0DW#(!Uq9AKJk;gp=(440@}( zvBPA&jA5#`38YDI4u5O5R1j}JR>2&$;nS>ADTsUrABM`SXQ>%^jD56854`T#K@6K z4_BGw$~?5bP+I)0Vh<{hf?mZADtGj8;3Hhn`QE5sW|OpLa|O;RxH(%gYU}HXQ7}58 z^Oea$G_k8&Eyb7Q);+29c$0KDpD*4Tvny{V8dCB_i4L2u{&?IUjo=!&p|}U8bJ(Oj zK~LeI?!yL=Ev&8G_DOxy`Syyl+Mg89@ozFvWXsh3=4X``oUSBQP!+LbspY?MmtDyu zmiLFmP&*!UCs>FF5H%GmDr!S-Tay0D3f{$!RGrMVlEyTafGy_23v;yI$Z-$i#_zkxON#OYl0goR+yw0*jaKcA^E(N?qHkii*;8c0YwqW1~ni6nqk-9Qu2rTIVdpa}s&8 zs7S{7x%{1JraKub0Jkly_n~zyqL!a)^Q)%%17!I`9KfnWO-NbH1hcnQSE$0yQX0M% zeX-1XUq}TOnd7+dGE3gH*wJ>-&GCulUukYj;D54_0n=16`lw3-7n|>(yc2n`3eKzu zC-PjS^M>B3x$NK8F+or9I(ZFe)$@zr%i|Xon+U5`X>xe&E=v3{#o~T7mU->$@EVc$ zn^CAqm7ZdjT%}caJrx4u(%+n9r~=3RHt<1HqRc`oO`yS5PsQ^))oZyw<-U2) zuK+XR-5#mQ`%sXRV;GnV;*OEfPmFCo zdqk8Se4bF2Q>8$C^s@6s>0S_i32NMGmW$(aFG?e0Z+`LOmOJq6NbCI5;g|75J@`ji zw{VLwU~Eq^xXVPl)^|ZW+G(^|v^f;j^R(1;j_f-aap9G)0qo)5wfBK$yAw3KyeZ|Q!ivWjL0U;Ee;k59)5># z=1_9=wg#M)iZB{9dsX_Ks+L|}C)@!SbAf_+47moZX$Mygc056|6(DuU_2bb6`U|6F zR}>4zg&i13dlK{%ziHXI6u+b^_3hmR37EGQtnu=C{{A#i!sK8788Nwr_XelG4aqz^ z`!^IDl!{oINHF_Lqv?GDN2%7k?IdE63Xx#=!KtKOH;jP+^~X zDO=0ec(ioaKD5{2^?!_>=9gP>)TV*yLsI8a3d7+bE&%} zDnYfyz0_ecvyttwTMlCKIq(#Cn9oCShBWN-6Z`bW&EU3OY3iK$?Ap9nlCf7+D5=z_ zD{Z{{hfI^zKmRR>S!cBh_JhNrGiD;9#IOgutQ0Fqw8>^=lo~`s!U738fOQzWq}i@{ zWH=3%7ZnvD*tN-amFo@G#=tF}`r5v&_dI_eO5j(`UZQ3SYxRsF;d-?03goZIknwBP z6dmlrTsJz55r@vc6DqVkiVrI9XYPcq0fR-g?l-|b{gI^IR;!;Cr0j5x(0&(h01s~I zbceNSY>I=nTV_>5cqDY*y?W3cFn?!R_=>@oE@h@=zPk!G_U*9N$TXlt#2_mA0mWF3VZ5Q3$%=SG9*Bo2pSCpVNn)M}#ufuO;Qh z85}!{1{4@iC-C>5d8pezCS_HuaMWnLlJ$sxHpw#-hX#=TQv_}dy?^9JWS%Ou8=K9l z1eA(MmDUPRE_}V!q|Rj=wz+TgJX0A>M|qy|0GH@2$zoM^lkNKD><(rY=ymVHePaNZ zF=gVzH{0$4ioZmV{x)uB=ck`>6cuo%C+UjppXctf-A+4hnA(nm@4cIW zD(>l*OJ@PCrR|j~qxXq&h>nHvK`+~Zv4OVJWkRt|jyT3tG-4jZWrcyd!klD1FV!VE zX9lffBdNxLQNBc~>!nUXOHK{$w|K!a!K3a++>Om{!{s+iUt!l&sGg7`Ug!QAX?&lj zvK7a_NB^vkdaJ{eoR`}yUM>OW^3^#eBT%`lTQSeg6HgMq|4t)Dps9KOxHe(?@Q+6D za%y7S*M>t&51+?rw}xhPxz6gTAKmH;t7?U|OOA`6(ox3?6cE0P!TdyuWU|3_7fiIo zB<`1( zoBK}ii3q3f8^OPMwU(RzawBj@RmiLhVHBh=#inz9U!Min`AS_3pzy|qTSV&d=@@|w z2i}hPXL4Ixq9lryyuHq7cS!uU-e^vj3B)1W{T}9+`;p4#7yJ{8->L#HAEvBH zNGe!+;`&>Nq)6-<-$B_V$C2yv&hU&ul2f&c(;8mThW&bXXhhv-ggLb*sjTJGtxVCD z!eyUnn?#bOc~pAb%T-;`Kuk6)5|yGgp+1r~0wa;`Y43hyXi5?B5gD`NF;%#;{S3ok z;l9@oegEs>nkpjeVMT-{+YwPvzSeSXI^?|bK!np9@wV-+TXB8Xtscy)RBmI{Wo^oc zx?Hor~C>*udLM1UK3^IYI>oo@)((33oAt$Px4}&j+oRvtEWprGE%$*NJyBJK; zf+wpHK>W7YGI%q=|MCp-MLKFynAFY8I-n{ga*UUmczMi%wO__t6-o1_y_~JqK(>$o zvFFd^16fi{D>U=Xd{VJr-a2b*f}K0vQdK;VThJq~OQW?eEM0GHh?@LoCl%#kmb86U zAXNn3DBtTW;X2G=@HbGmu7E)y1a)t&#ryc^m<2`~-gPQ#lDV=~51r7Ecy8`0P4Hap zvO2PNI1UBezr6Nv!jczVV}Hr-P3+1CD7Z3oNRi z?q811>zJEYL2numXC<`Yr&nj2y_U3qB$1I~&zg*J$x|3|`2Gkp2dREC=?V6nUN_O| zwb~q}R75ab~_5}6*W)&^%G`6=z^zl z*CBO*btu98JnE$?Bz?74cnUQjMCpJo&~JOqVpL zN|niGyF;shx#iE!i@hvB%Bj^7%PiubHXX-GQF^iI!)hhq``nogJp7jaX88D+sHlKI zGlZZ6zN!h}if35YP;JTI0y-Qf5z=Pa zzBmkpA7uqj{`q+`BzHn)jqcA@nh0rj8ZW%vH*<2Wmrnn6v`ejf{Lx4$c;`e`tCD(s z9o~m}P=YldbTv_Vy4Ffa4!qzp=B#YQ!2AP5gr?CsX4$s{E-?~cku`ZH)R3{sppVfp z^gG{Qv?%-dG8x8n;V*m37fAQlqiJy0J(5@7{r8C*_!2wBO72R?|H8i{PL`3u#S>`X2&ae|d`h{b0V%OMC~NlAx* z|5ZN@G5I5BTZ%hqkfv)ZnvyOaO7+0~;Fx?^vCosCrEgfN$YmsL8ObR^q}bC1re^GI zj==qV?v61=C>yY40WPB3YNkYIIG%sMi|lH}7xOM8J+0q8M&Rx#nfL8Ga&k?8srdyZ zgv8_a&>^Q$SDlGvOPMR#yKR8Az|uKz(1#%ib%csrrOuOSf(e-P=re`C61aC7lGo7C z(9^r9)Px=2b&l$rYeyEP}VDbm@e1NhW&s;KCOJD7^h-O`ts#8&+srNmT3s)&8uXL2?$7y` zx_e%|Ju5DO1K;i8k`WLRMmrL+y%is;E-08ZBJ|rH_kXL- zF;;EoQsObwk6XEZzXfan^aj8@?vN9+V9(R{K+*a-u<^py-Y;<;A-=cfd23w8+lKqL zXt|+D*unGusL69dR`#?!tb&F|M%1OO z2iv__OcM4=lp06>)jKSV6I=Ve3ICZ~xdyynSINqf7)i5P?a*($BIxJ=$H&JVFNo(W zP01RXI9xA>AFJB#yfFnYTbGyxMnuLtAJS53vL4s$e2VV=33%}WATnwH(9qEL(aLt{ z4hB;0a=A}<7Hxe!-?*}<+k42IJ0Dai`-MJfVkcu)o#f`?zBAR<(27b5W}5mfOZ~%$yyQ=Y$l}Zmzu>2B zV#h_z;W@SF+3)5#F=q8q^>AkpD^sy_rIeQ;^N8-{tgLbJY3sj&{9`{0I-M$-n;mI! zqzKF#8HI%(Ci@%(GLbhE zTfhw2bU^__Vb`8SOnj_oLk6}UZou6=@0f`#hK~C_o!xA(qKJ|VZo2j7NA-+zB|Q|0 zXMKEG`OwRuczs2+*3L@RXfc2-I)uF~VVn~2J%NPGGnAwTj-VZm1w2f#MW$>&Y zx+>NynX+mlgp*uI*6wwyKMLYDoGDvrv?K%ZmeFJdu4e}Nr}%wv_~Qf$v88l(Z8ran7{R^ z(xv<{S{0tHiS1&wMrL<*YTf(EvwetFEvT8Mb-$vsN^4!n~GCYUY)8_JrSCNr*GB%wkL`%LZ)F_y|5#{Ho z>pO;eDEw7FhBllecn-j&eOMU0nf> z)4&C3P(bnex(0&u zDxPG@?DjY`RxItC2lBD9>h4zNh}yfRZm|C!O;;7xR@ba?E$&(zf)*`MB)Gdf#R;y( z-5rXixCM6!#ogVC7N=0$rE=2mf6iU5b}rW1v)_4ThWW=clU`aeZOn-hX@2sZBv!rA z-At33oN*`I(owNl37zx^G7BeBoxkj+<@z6rM***Pm6S`lIoE-mm0OJr#5(ut)s>#^ zgZ%l#!!kH>vQT#wmbmyTD4d<5vAIZrAM{SSZ~tiG}lArKmBR8q@qD3Diw*L6|j|s{6$aJ#dj5e>!$Jb@Yc%TU0avFBdC?wQK>7 z#Z(=v2Jk+66HGKrdL%4cI;KbgKWz^U>Z#0GI*3qTqGRzoV{itBX~k=0p!CO1OSrR2 zHvFv@8r~$`^%J~KOcKq-Pgh-hNaM>-Q+=C-|0@@2+^)!l%ID(jDsnV28ygKt;10VGloSyiy%4x^uP?chgIe{t6pZ7JQwO|J;EaLyUfYjbhEvvS!gXI9E<`u*0 zG8k(b@TWj6-)ADH;OXo?rF*eV$-v()U7W&9$h+sL&G3)-W^s zyDI+oD6QJ%X)HLPtgbGpHt%5jN>}eO`uip$u`_&9p0A;S@345j586Mlb~9ir>vowS zn7=)>`k`e4lTHFN*(S`%Fpwoxed0n z7iBk6i=l4_1&;IYSGr;`MbMtheWupF%8THYd#dMP&`|th-83R%9e9uKY#J<28xn=SmH` zyzlQ5<@p?oxL!hXO?6-541rt+k@Wpvu+FcxqQAH!Cf=w3ubPz25NJyAB zV9xmbMXrA$E%SRZ=0dR1J14R5gUS7(5STpL*!sSlbSmF*9p=lLuzvb&xo9)B+3Ii% zYmjLEoff_35u*}eXxkd{2$4Ikvh(buzs*$qm#BMr2UP9{QTH9U-9O!ZH+Lg71txte z_T0n)Qf$;(#ZVs6rY9(mJ$P5Nd@72hRD-OvTQ_u+(0CC)pXSR5q3N8HGb6(FSDv!@ zHs3Q!8!IxlrmY379m_NB)PT~X9^%wh0h`Z^_EdI#8o%Cj@~V3J>c-ldze#$HR~N^} zxUQGT3?m`0|6FoseEj*(c(1O!9vemmqpA9}z_98sC=5q%r6FgatXf=U#3CVGZuRZP z7~O`9`FnH=4}Gv%CVwqlZ=3kDF-ryDH75O3$}-5n^SPbS*x5%-?j?!HY?fJZd@&k% z_0(6Lc5{E{$}lC4NiM&Vv3``96SDRQKvG(e_eh(=eQXaaJkqkucre@sYJg8_^nDDS z@xYl)L4Z@VpPD^9pFE?o7hk=>gF&W1Da~KCg>4j-{Cu<;`VRNVSRX5F)4vp>SK51= zr!5zslh3OA<$y%q@c^1Kz5HrgLQ_=@nt775F*h>CH)gaT|11-DyENgSn9r)F%TzM4 zsq}e`p*Yj*CjJ+B8+2tG_2-G0{*yAJL_R&|tx=+;4os8hjKAs8&=LeI&dconu3Xh` zQVVHg75e+G{>5=f|2Z!INzsnvNb+YxyJZ%2P!RY{dV`C0wxL#}BE4xoIxpvJOzSsj zf{)4UG)O{lJB(lC;2On?r;$LCStyzh2e(>}MK+g_lAiPd!x5s}<~B_`D?c}g*_@M~ zS=>6QqDGxqhP+}prp7rY8)+nwnd&cb(tevJ`nz^YZ`aoa^JK>7=@_^!)^*3Doi@BI zzny9>cCq-i%}l*NCY}us4;6iVKLP`6bT`F25zLY`aY>=@owKjseN|}FUYIgJ2bBk| zrGZrOE_*Iq@&nqFQ(3(FYI^%*+WQa7>Thn2w4cCXuTfsVP!?^M6?l1KUC=TW=f@vG z&?`6|P}8JTYKA=GZ)=Nm-|9O>oC~VofQqMc?Q~Ts&6Z+)JC(w(57)}k)#&h@LvmXN zi@LzX4eNUs9c$gv8yQXCx zPN&&s9}vNmzus5ot%U^+Ovs>ep}f^3nQl{heC86 z4T;as4BOsEZk!OZ#c#rl7K?FPcqz6obdVv?9AWptx7Ca4n+^ zWA;iEst??Jz3GYxL}fzF;p&g=w=U*>pJ%*hFQFU%j_-T*uBnQ^F99tjr$pqMvKh4L=Oel zH7$oYLIavkS`D|*E>UBTqscEQ502rMM!d}S_gX5qLa#l~f>*A2O@MsDd z2iGgVE^`G}SyV-GM7(#hlmVi#-!{8KH79!!zZmIUXvbE#pUvrCU_Ty-aZm^Ts_tjux~Ve~0l764IqJ%z{&nQOeK|s|8`Ay<>Q)y81WUSiw)l?dET&&3C(42FSmzB=E)-}b z@SUoqGpuPyLb0;H*;I`ox&{KLhj2$|OC8GxQ4Hm!6DuF?zadbqA_7?G$#U3CN4MR^ z@*ABf&JeQuhPBz2RQ@zqZeFa6dRJepi~v7{GMgkLr`Q55Q%WF*OLQrix~yT;mV*qk zqRc!{leR#^2-yule%T7JHUc6OW)KB;tjXvH1QKQcQBHjh=9C=Gjj82vyMtu!J&q>t z-JZl&@~$ZsdKJ1?Ai=B1FQYQ|>I&ZysVTPkNJLCFxK2y^9`v?U3I1*;GVRs1{;#rF zg^Whis*@wV5IYS~kp(HnDA`|z3sB5|1*s!(2&bZ+Hs5}2(qCp4Yd-GuN#@MSWI z{beaMYuumrR8+#hQzolu@G4R3@ST8_R{eQ(Jf8^f+FNcRZEA~70D^*}wC75j zN5$rYnt1MJ@s`jEZOnD3|6rmm?22#>#QW)(0d@HP56!TI>p4@zOo^bamX+_sve$H3 zW%<}ur*UoBYiaW%mfMLDlp2NGj~t^4=_*Ma_*%+7&c#?kAVnE7pNg*o1;z->jU^KL za(AR?yLDsYoQD@mUnT&bHI125K&aH`HSk@$@-`Hg?<6UwG*D+kv~kf|39Dsr3zX-r z77L`-l>$H9BN$Hb6KJoe*!c0!Er?nu)!OeoT3}J#p5klBGL(u}Jb8VdDBhc=M$it;wBAYmn=^$6d_~B9?|A>nc~vBp4wQBOkqrh;+ah zcKRv8YdJ85??~X4iZx#R0M+3W1sxoRr=YJ6tJGl$|K;zxMO*7g+ZDAh z)f}-`1pu7dU+GGrlJu$5NATtt z_+kq%eoYB<~RwU@#1B@s&w5=^z9DfGlr8>#r#EUH78%EA5X^4HmGW`B@>W~*h%A5Fcc zmh*y!Fy)P6^gO!A>Hh2<)zcL6Xdz}i$Wui5967H?7c1sudU`e*b z$btCh&S!zR#;j3*>+;_Xi5Yvd$!ti)iYhQ6BU8xH{BxVp4X;%Wb>mst^z#*lvT98b zyxVujq(i0Bd$~GPCC6aX*XO2chL5P5oQChSGVivsG?hagBP^$arOJm+ja1&VxR^H4 zkk(UV?a^_bS<~jLEWW8vs~|!sK7NF^=UD)m8T*zQ2`lL4YP9(Z3?Brku%&nAaP?Ub zns&2JED1|A-r6p~g@J%+Y-buH(v4z2X#s<05TjXkLk=V);AM?5Prt7|U~>mO77I^+w=y zFTOZ8tUF-VRIN#Ad-9Mvl_`zinr6T8hiU({am6$hm3T`l%NUE*FW&=~K1Y71>(HO2X@WGrAb*bUL3{+XafOO%&e_T#{OQ=A zbr65+7z{_5w1;qb4hJVzWFdbxX&^@Jvnjh1$SwROOz4KMhT>OURMi`_yQlSTU|sU` ze9W8kTey?Ig2u0^k{}mF-!CCEaU^b6iYeS+<7t0{ zAI$*hi>Q?Jpfytbm$e5)xGhdfldGyuQaPG0NBbdXtonaEsAs&sr477jMEjS?{6-t){c5zEJ{JhBL{LRB(`M@0%NnlYRKgL!$*$*@8q;OLvqjpo~vh^R9 z5>YM;D16+U(V%4{VA%yGTBXAgE(f7Z(=x4+e*>sh_(&8w&FRi2#t2pvsa259fO+hr z&>`C%%wlzc5{G@j)HM1bOldss=(k!6a(haZ)Z(z*7YbwC1VKkV1)tS?GAG?C3{Utu zyz`~WV`PCW)%Y02X+>0W(20gww6wqx0U9`)S`-g0q|m*_3*tmh?m+Q*cMG7H)qZ8R zgeCp>s^22*F9o`ei6sfwW`;Vs$mJY%WbBLoK}`)pNBVeW*I2qaZ&Aj&fa|CifQ8Re zCZ8x@{nRQic)_RV#Ne=Rzo~P2%J)sC(ZdEtQX`RqXEnpDrJ)Wb@`zD@@kbS@%`1_4 z(i|kd2JHpXPZ4q#=&sI2TZI}ICheVAwWvv=IyR0|PWkzCpTB=?AtIFMuifCqm&-_w zFAz8NxN4zcw-(IDQD{6q@$PXlH14lm*o}^wN?Uy3^lg8LPw9YU%pZuanAu7jF}5@x zf1oQy+&we;t8#N4V5_3p(bUnztj9LRiP~M|sK{*sa@#6<6&j{|nu$5kg_}A& z6^;4QvQ|ElTRENh+d>f`#zbVi%#Vv=59u6bAs^}9eO?cqOLS)5Sq$i3*yrI)Gjz^c zR4CX~#Usa*^%aSy7hJvNy(DB?coPO)V4;0BrJhPf>SE3-UPQTABjf-KpAR_F-+W9f z=|VD!Q*7R#V<($C^*moTR17KRL*>_M_iSjLuMRbT&5J76(DCMzsqb51wD^%m8~%bw zofit*vc6Ew;bWG)fDy(oy)0j)OOwRV*dqB2Ywr!~Z!l0rnVi*j6PJ;z_o9yOtu|1zvuhPrC=um6 zr)M$qe>IiB+j&r2|F)19rr%a^`SrH4F>L+ceuL!KdVT!fuHxvKe#Rs5nW+pRx1~)Q z!JLREKvtl1K3jxTbjr|U()zlQ8Z+n7nBgTl_X$EI=VeeHA9i+GVf(m@v4CQ=cNG2i z0l;S&$#}GH+vM;?8cRDvSK>rj+wpP}(25TWo^s`M-uzWWl*0}QsrQwH9jNnyup;pT zcSDL^fH`D@Jk*2rB8TX_Gwi5%-m-2YkA|OCSlT}WF5iQ-SZK`y`=*FGY44Cx#Ys`ivmJz6CrcDzRMyU|y2wpO^cY@pCOA(mfv)F`3X+Nt{ZW_;n`SB9& z>{PYahlFsn@Uy}P2Nwh(7VEYq2TQC$Yjg~#IC28IrDiQ8;fpntR+H|tm4p-1*I@C@ zNqbhviHM0!Zxi&`3Vk+r@lI0_uuCUY<>pXU3db>FPjzDYgieHzhwhXCp2?VuiSd?A zrrvsdTXB14n)=5elv0R6`W3jIoPr;1drZobTO4(EGsO)NFGos-EG*u*8$_YFm~EYI9%Z#RK&RCSPOuYQN1&st;W#z+IN=*ILva3R{`s>3L4^m=w=AHn z%A5+zWv#uvQuTGn>Cw>X&(QfhZ>~ZB_kcJ?K0mcVL3cw-x*r)63 zuXUP70g<@MJ^w+58d||^Ww1?xR*WHSN;SE$f9y^9+$mbp>b|mX*_R0hwi&xEVOZL? zx7->KsK@K25W|NAcCM~ojRxis=A~I^X4KKETCF9L+b6dm^L5oCUaH{)G<)?+L<#+XYkUucp+rw#i2+reHfm{i$Mqu_{GDIZ&JGSUkSghuK+dr$&5+Kj zlg9}b4d+{B4F+^n^1Y*NzUXRE5|s8*_P3|5+e$^E`+ms+INZ<~l>U-EzNBI~&3oZE zh&%$|SmPaPzs_a6U|}FG&8KGtRXC4#f@&?7j&bN3@0Rv_xH3RoG!tvW$*`FZp?eIp zPR!;YK`slq&#g9K^6IV35F30W-g^K`S?G>pZ#k_Ak8hgIIouwqkg+`;Ks(s8x+OrR zmSbn07zDzsrH%y!iyQ@%DG%2BK1%KiHxzcpBKH>Y5|hI&a9H!$W0HQUJw&f+{#r7 ze4Yy&Ro2F5&60KQz~L?wgV>joph^or-m_VrUY$6m%`$Eyyc;K}Q-o5G5RirAl)ot!#_ zXv+$zegC^h=hHVn{Wm>RU=*VbYk61!IU*gRHVPH`)5Q4WTceAuK4I1AP)*Q$ZXd7w z9Mj;w=lYs#nxKfx7JMRqI+FGG0Aqcj;r^a~Oa!)G8mm9rjypdAmU*Hq#~1YNROsH$ zap+za$H3^a5D^YUAR4#*R>sSS^IdKTc9wB8EbQvoNwpW{uj-C8Fpb-A>8@h$Fpx;i z>VVOl6{2I|jQ*-hVqF2)Lmvh2fRds&4E3haGtOw{itGk|d=Rw7!*jxNlSBqS8x8%e z*0_qW&7>T<3W`IK!D=SrCb`SD;5kYLCLI#Gyxx@haDD+5ERYhm$>=wPZmq1_x#0y2EvMqK3RMZbz01xe0O$4pR<%csYy>8ZN`*7gE{v zY0ugT@&3+`O3NwdfP#uJLPpqMnH9dtoCN*WFVTp^aPLeKH&a4NsYCJ2m_jVS{vAmg ztMkQ|&$*BHF{Z!lon%}H}KQeMw+A>a-C^IL>h$2dhH zXMnF!1Y@8fmhtDZ5pwXGrJbZq&3YUB&r<6w*!9g|unman`Xt-#s)E(i5R*PRr`)2XMSg{hhxq^Pm>>By3t+*PQEsS&&TC zG#{JC8Na!pMu#8a7r$THCSOi7#J@l3>%@Ea`{VL*Oey?Li-gYyJ7i>~uD`2k(C>I^ zBSA4g+u6Qv$FM0B9xM=~Y?D819=s`G5!rLSC{T+1?X)pENL|R1)c0eAj4bv=(pJ~X zI3$4gta&%zM6W}Kv3x@!8KvcS$#;VIFIjG{- z5_>#7SN=$lnbOO4_6M;JYBRwS_ppw?xfpJt!WPO=VI2g?k6j$|qCrEB_tZLj#H?^rV7maffm$XbV>6iX^e_JEnIJO>Z zJq2B(poBd=s}stxLw7j?Cu;PU1ztm(gddd@bw(BJC{C9BF8=5k?-|BY=_Q(RBi)R` zrelwPDH>hR)facS8~A*1Ymbm(G;?(#2k)#>mc0MeTIZxzQ5pgqEt=lH7@LxjPDi?- zU#1Y4RL1wae_e8^E2K=bY#!GHfpD6$gFLO?9C{qBRCo{G zaRNlLX&5HjLf3GDr}|qCTS>>om%4!eaDw#40M7*FH&>E-^)26T7ljFM!`6?;`4Iih z?aDZ)5AF?_hysmn@PT{|W&PvNT&;O(@N6}M*sPYgc)A|`g;IEA-ua5|kj|B*BB5Tn z&sMRX@sPD08n|=zl0TF`bKVqV{`RTxVIy>Ni!_d{-cyNrJWf91Xk{%OSM8p3vBLyE zUm~yG-rC0wkZKxP8d8%f*5rw(2fUZ6UbK;;w8PuY#0N*n>%_HKj16Em`~?^vz8~Vv zK-?AR<=Y*pXNjIYf7EwN1O_`%NJtO$qqa6v z!!3PnDG1M+G!ZHkRM~|ef0WLqyp-pe%>%{|#4`h$h0IkoP%X|5AFnd4Ciqf?u;U2s ztM_KAtD-W#SL9I*U9 z7m(5}gIi5>g(|epSytIy&g9>k#iE+9h~6f!CnvlvKS*A&Ve>Ra5aGpMsFYHy1l0Hm zv&F6up9*!tMYUyP?9CJ7)vMtL3Pk?c@;)&PZ6ROoDIro2Zp8?rby88UI;jst%TsZq zjs=OkB>w!OMNuGNwQVT7IT5$Dr=I9Q@uT|Zzq%3AhQu?0)FFpN=3LBpib2EP%7yw= zQ*?fB@bFgslFPDa=`=|?xN3!WNr-4#R4n{P-pklK*h)Uod+7REM$I-~rJtJI2fBLO zvR6z;3~VG7t+~vNJ7H5aBT?ExKK}3}%q673a)J#pVENJybDiIf(!o_OtWbn;ynnP6 zm9v!kg(1H!Q~ONo!4(D-zHVL}Pdk$R=@t1xC)g^KVf?bN^tOHp*teh;5%!ap?!^(~ zzuP27U4PVXh+5o=*Kmx^f)zurrsKZ^qc5>D58eqr`NW+RrNGY0V0n{`SGJtWj$8~9 zCPcGQ&47jtQ&&8;1fXdnP$>`B4j~&_Y-Y~rz8F)tPbff$I3hDP$up0T%0BKM;XRcp zwt*lA-KJjKMLkRItTba29QKSy0=QL6(=iF-pl89`A)0tpV(mGE*q5(%Q1zb@Q8G^r z`0->cD_uJH6ougqC)p)kD5l5A&|DT)jG{NUe_mln!}`|=wvR4?JWzoxEv0NKaf1ci zG%O|hy+jEy*sWv%?RY**ANWTL1UAyK8Me}Rv)9t;K;l~1z*}10_W?tf>aUE9oh6Ov zb4wj5VfIl2r|-Jg_!sFyBu%h+RS$8-K>Dv2@aNtwo5T$ezD#w}`#-h&7 z6oalI$_5K)jsT{rKye%Y;$+K9Dus@XDSFK7lCasV*tXZh^wHL)JIUlRe_zA1T6yRG z>&!yCJIfmpm=9n890lcmn0KH6<*D6-@znzDVTsj@VU9loNWp#x(DuNss)qMl{)lbUGTZX zmm1kaX+jPGH(Dti zN|YP;(2lDTgXM8ZD!mw8Rnz#G@!j$UO6lwuOmR`eKj5dBhBo3Py_DAB)rcqEIbgdW z;YtQMNp${#{pFrhm{@03sRIr?zw|gqr4x8Xe>);!D|jHu{<{VQ33WbanI|HcaL`LFq-!_L0+<5Jt3 z`x}s%fWGl-q_)Ry&MlP1RH7gM)jt!mds1X9UO*%Mz1@&r6h!x`c&NiF*LOgJnMRW& zgE%2qj19^z6ZCjuSmK<9EG8+OJrz{haD9cpyvA|TMJ&vnV+!t=O#GZH1vi?Op7WmUO zn;{osbcUEt^wmwHX=j_bY5l^8%6$5remV*5@bhAHJru$~@BRO$``0Z-rQvLvFqD z=4mmiT2$xJn-_2CUJWKz|?ZRg^QyO&ef_kYA@BA6bi`?Jb zC+_Dhn1v0XG`L2q$on2M2i$kbKsZuL{wzRg;B6@vS5pMhzBBeMK8`q{2px_aIgX7J)l*W7@Z>`(6^ltj)qYnOHMY+Du@?7*W<4Gf z5;L6fQ0@>WfMUMVQ;F{Dl*+F2D3MRTIx^G1K%L(`;^o(n5mOELcSES)7R&ZW)&(KO zSYJzUrwh~Try4UBU}A-nnm*EK@#a{zbH?$Exv!_#X9UGwntd8hKjB_{|i&b z4LzCd`lp?1mwCbm*jlUV{H=hV{?=YBR?gGd;UB)W9;`{?71bsOf?cT2QUD z-Ck4&%lHkuT}2Q)Zlb%9vyHD8#*RxQ39D>RxKz-1FnHD=&V9s-o~uJY_3TI%}lcIBc-0nkMdw@Cl-rkkC1tn$dY1T_b;`CUG{8MB1$0%jGqmehPdGG z^;K_ft0XrLmJ})|d1=}^bnfRfIp}@`oIKChB%=!up4b6FMTFt$>Bp;VYC;tpAn z3Y$7PJ&n{&#K&69!>$V1%eLWkcTU!*0nId4UV$gga%PY9g+%iaqm41=cBbD&5X;B6 zL>abSAzzc!GZ4P5{G^7pR^BMPlEqAh3Nr!$CwbO#;gzSD(}Z!UW)RmXC`pZ~o%fpA}{rP>sCyO_hC!$Gb>i~ET)yaH5I z5ocu+Q)#bR-;fkqq?B}bg1_^bCGGaD^0b6sc!s<^TB#l8wLNBG827QRaV7r{D=WY& z6rT0nw2a~n2%(iv0s_xY1>{%N0==~daR1hz=jC%bcj#or)c-?V;}&;HW$>nXt z!qw*cJ2ND&Ris41vqlu|5vo|6*b#vA{bj}X)gII*hEjLN##i<{_5R9F2a&R%oez+( z0HL8_th%>9bkVbuTJQMKzGhUEzRH0gVss8k(cA=6*R5gmoznKk#r|0s{<!s3pkdl@+KiGU!lFf4KWZ zR0B0TDp2C83jF7(FP^q-_DN*mdg!hM#P@ckU9Sbv5@MDs$Zn+{TcY+oSo3+=P*8_u zguT`SatrwgGHk-*4KStBJnI%+TnmOEsC_~Ta6S<~$b=9uvdZO?&iJyPy@aOUs;UcK zM{alk;8-Z#csJkTe{Is)?a>K#!}?R4a8FrcRiVSAI&K9RH|O}q&-f~r70pkp;{mg4 zwLL^A60;!66jIICr4of=E1B7HWyh$79YiQ{Xp(}2+H4k84+lg`U$~!A4SY1_nT&-r zwAJ1gqp=t!0&~r1g^+ngRES{?6Q;yddy_1O**DXqCS&kMN<^_rfQUUQ5gEs;-{OwDbjOR?{)VyGE80b5_0Idu5_}THQ{9Q19{JlliJiY%g05d-sFnrnh2(zTas&UdYc4@Ax?n zJ@i?lRFE5{_PNsP_c@y?Krg5Dlq=hb-K5y|oMeoV>4W!LR;{iQ45OjD->W+O)jLao z`C4v!n!F7uoWsk2O)7%5V{HVH{aae3b_L5M@cZ%7ms%i-mnRk$Kmw1jAYa{T4a*{S zvB2bU@s4k(JcrFfCfk;9&XL73IW_$ZXd*h2v*tB(L zG-H8pZ?1C};o4_PaD9tk-2R^nuss%2hkXK$-My$P)+I*j;rCK`#*3wZxgWYO@&#M` z(Uy7y`hC$Iu(JuY^Eb^}gbJl;l1eB1(<%rnVek{^p9(ymKn*8o`9yQRm43mAAAj2z zdO&02a$SjF_8?NI7@wBzU)ngTSRg=Se~BlX0oN0D?3Xd50Oujrvyl5;B20%!x}nFh zu4QS?6qq=*vdzcg#n0;&_692=fIPd{zk+dHmn_R;7!RX%Glkhd)}7;(S$n-k~>>OJ)hNz z>quHnDHNfsZ7t#YOBQU`M$aw6_ zFmzd~pelN-9l-yQ(6*TNj~g!XtWXA=Tz>PB1QuLr4$Zx~`>FfEwdTP!$ZYH9V0c=H z5RE7hkZN32Xl+NcJohcBCTl}xCl$_IXfdBDREU2TX29|j@_~-!?%p2nDhR98F@OMk z!IoA zMEJ5<|2ddTj{!u?mc(2-v|2BS#AcM6T=Shs>nd7tG>ViRCRU?>FO8ubYzeS$w8zWx3tL?vpGpqERGHrT~4puXlJkt%OL`aC-tNZ6K z$-YQONwTv$eXQ2>vrhOABM4~<=cmb;eqHDqVq@%GfE*C#xgJW9(WRt0AfjXD4N-D+J(7H5g zq`_KIwo%)&MaUtQW9|5h(#Z-3GZBh*cqLkHvQUs8GeBU)Cp0a(pA>>XJZqiyTO^w= z+_tyL$#*0j)k6h?wOc*W$I374W^kF4c4Id}5l#9}E1TU;{pVtjvb44ru42JC?_YyE zM=IpZ9r^5~uLlJf3APq%lD>ftPg%Q`5XGpaK%%aLGQ=PhqOp zs2$&K>8j#9QXRl@tf)eQXw3oA#fd_X28ij`dWLC^saL!FeAi!f&srYy-F?R zlt9hF-B9;WrbR1)g;L7vSh^Ubb~yP)_+qM7_=X{~4|SjxnU@t) zcbfRCY8ftX2cJ$nkDY2^4hXPH6TZF&iUARS1y&Z(uC*X6iD~&d#4)M2IN+bVBDG*@ zVwK09^cFMGIyN2g@J6-HYg)hW9&tj~7xp6muUS;sx#S{5?d)Boq;yt1Ak*ZiWK4}N zEU51|d-d!^T%RSG)$0^Z5OV1FkWoxkQ=V=b2l?bNO8DX!cO78)3xQQuT`cNO3eID` z9UljE_|P5>bU#__Hzz-FwuXfx-GOH}#9YAD(KEJdE+`yTkH)gO_wBD|$Bl-`cb&_1 z4Y6!!mcFsH&VeWv(iA;8pFCa(;!rIqADB+f2!ycPKVo_Sa zqRN-`AP|3$`IaW*IKw=Nj;LKOZ-*?j++%E%H+L$8UF0>xX{sAQ(oM0)_uQW6VCu(b zc5+yc=h~sw3>-9s=ls;FWM__=ua4foXwZhU{-Fndk4Du`*7FagreI6ThoYulT@sPV z8j+F%mWch|27(rCNyP!ypi9?BHxM&3$y241{EebnWa( zljP^M6PyX0O?NFr2mS9AlL>i0&4t5>*k+<8Km4ar3-iWYK(tq9LKB1L)rgne@uZE%s=0+JBr&d&h5~T; zg&JSzyGcfmuhK*U8-H4+G>qo=2CMb*r0Au2%PDdnCfD>q7hI-qNf&3iSn6d^5l)onVLs`GC+hYvK8I&vs)Mf%%7k%v}%>JHa41S@iJ)bzA(^y zwRZsuzNl5&V}rkz>PefL`Q64BALS!=p|~h~^Is(asN0du^?jEJ3FqHhli>a3<#>ZF zgWC=1J6J%`u*%gIfxQ=R^m?kc8~R-p5b;?PYQ(>Gr|W_%XonDaDcgm z1c)b>P0D{HmZ;&sc>LWSwSJhoMS{}5qvGB?%Kh5F$m+{dcwpTk>#pAZhPw6YJ-UWy zHpC=L{m&0l730>jSHjy8_yGYaS zZt{xFKHU#1_PZ-5?rY_6MmE&El#pIu8o!zjZXt0szpWV>ql@xvRd;I`sAQfb?S z15eio-_4Y(JGnTiHrSB!*9rbdOM7~=a1A_6pz*wHt}`2ZF$p1+JJ{7+&%qz28l}LtI+BOmqIf`UbxhPZS%Dn1K$KnBB!sG3OzyRd&7fxLU>R6 zO|i0>wLR6G61qf!09;Jxo&PYMG%IBb)2Zg;<}1R4F`sclX1yh$F*W$oI1vht4ym2H6!q86ExW^pWx0N zYe;SiL(;CN;>hZ2!b7%C8*xG|b1{UAEE4*E?|T9yP8tC4WkW~bXa#1=^`)}-s6E5Z z_t50SC_lQ~!r|KqJUL>}sHzLAV4Gjup)PZ#m@H$h{`JBAdU~w+nJDYz(ft(t(}=+j zZFuor)OEl3t{qp5ZL)+e%3r%nIpz%m{axk2`~F8_54~59s~iu!i%e1#a!zb6SWeqe zKsMc;JZumvaR9tR21`m5)*y#^%b`lIl?yK25NP=QAYg5?)JZ?+6A}NF{5jATliGfa zO&V!Lv}MPt95P;@Oh2xBOq^EBULc9bsxW+;k<3L9;@7i8+1hi4IaRSE+@W2MA(#Pd zI#rNh%ss|VJ}Q+nZYn2|$62-v??TqrM%unZjSrnjWhgy0dK#}qgs}>VT`EHS$WFPc z$2r66qHLFuTe)0$&sqhLk>}YmK=(h$5c3eW`j&IPYdhY97erU-bp3D3p^J98JbUi+D%A9H;6cL*0<~RaSvG@5w(0-DVj6b>N*I zTI1h_Lphqj;QlWs@dCJN3uKS4xxz0+f}n1cElqx4&%>S5e?KiFTJHIb(H1+<`c1|7 zCp#iGlbtoT$$pNI+fpg?s?FcsDAFGj-;7J~O>ca?9kkUrqNTc-09Kkq$9l2hu>Mk| z7e@Y)>2fNfk~Bnm&j7QfAp9`V0J7(M{&44yq*0q+D=K(>kGz=7_ZX((go#Bl{kLCc zpz?s>Ihi*0kAdsSM7dl6>A)Jz$f*a__+?ZT=Zf@eTm5%|qOd zIxrIiPyVgjB3A<4?OnH1SRxKTOfHiV(lKO)HrvJ(^1k;oBdVx+S7vORM!7VDoT6EZ zS|9)v0Z3I^l`{~J@~$pr(#Toshb>VS{+XFy22F8P8dLZq;gcp_Ioe-<%F7bVQ8bqf zb?_S)3O{**g=L*oZcj6mliJO1KzJZ3bGxK^4kW(OdE#_Q?ey|(BO-aKi}WfEC!%J? zGGaY2d)W$-#Y<2AXeBwb>N&ytyr^z0fIkYRl=9(!Nr)et>MI*J(Grhh=%@z<=#O$9 zS8m1rI%hc9$J&&!dz!hYb#K($qhyKg-s4}_cmVao2IqZ-j%I|2M8uO+6LCfT@BvGvu_gp)Ee&1Ol_YFNN^!Hxg*AjWrT1(+IW8oaXm%&Co+coDJsKwU~Kw|t{XjJx@ z6?z8}=@Ev5+G;6ZKsSaG^^(sjgy|7}T$pQc!e)lh@(TCxS>_(xH5X@HpOm7@uMPGO`irO_#_9;0FeC20=) z?B4;C+v=X_(o-2Ut{}6B0yQvi6wKVRjhW!(S((EoJ#9!cv;u;LMIja*r$WsZuIk36 zxp0EDIv)6A@nXqgv-LhufP!$eHnzZ{;wjGPv0k6(q@;$RQ=rd=Xa~ojM8> z%+kuTl`uw8{vRvuRZ&l6wO*+6=oz@_67+NEa{&DLe4h<#s7HvT5&b90!w`|80S(eD zg2Y@U8SY1&4&x-sgYb$gI2OOSqacOzlsg6^c@?o3c*PhaO=;r2HGFNjP4|Z^2%hA; z`OsABbZh*NY4wJ&xT@&0oQGAkR#+<$p_yY7=HoYkAYw^P>*Jemv~}~Q6>UGR27C#4 ze^Nq$tnKO5s`h*^?Y3LFT(&E;tgTKfw7#d}lz$N6vdp;Ca{a4uuLVz(XDX#l`AY9& zL)v0($}iOQyrmby|9Km^9QE+Y0l?>Hx`?J`35EWst(_)iPX9IeLUuZq5nGz#qQP^ieF=Q+4#d5)b+>LHW@5f64ZLt zE%T)Dl|%p<(0x2Br=G!-4?56IGYgz5BQk@}8Ut(CHtVA!-GkA6qbk=NRd?fXWl@L} zDBK~e|3}qV0L2w-+d?2{a0_n1f)m_bf(9qJySrO(cY+6ZcMa|qTnBe|pMR43@2mHy z;xI!E^pWl@Ywupm{P5Jsf?Fi)qNrXGv7#nyiiY026^DEeCKPmtF8YX;!nVDb@ie2K>Atxy7s+$NZUv)y!2}(klj}L27Pj&*7O%q>&)=c# zYxJy6u*3YZ(^Beul~tS}ILnaO^nkl_$1D?OO?k#4=`%i+uB4sZV0ncW=5<6=M0kK!2~RfosfG`Em_95dy|wF3AGsbQ-amjPDOVUYZi?PP9=7-5 zTybE9K*6oTsyu9B4)GaD{x$SONWp{c2aGhtlFDg}NBujs!g+-I({WI1f)6Ds>U@=>qFULLQ!{I&qdCK*3Wc}%cj7C$4jNks! z4fH0Qrb9x6PhadRL_&f-xJwkrw^$1IPjWMC!z|p^XrXGh+C;?Hss zQR%0YTa(g=omq}C_Lg`(dnPvp^c}i%C@lFpW-cLT21P`oGx@H`RF6L>?#&#GL%|Q9 zMi9qS2CJi$Yy;pBT)Cw7_3V~ad3V)7?XpEXVU32D?zxVc0`La?RQ#sG=XU^eR3Cg2tAcVwH-Fg9<_O2v`^_tN zx7mYxlCg>pJEPch31^QK`4qV~Lz-{7R%j^Fy1|LGySOKAit%JC#SkyfLM7g%ze_W2 zlwlNC!+Eg@?7v!@Bxzj7=l1riY1976&b56UDS|dja9P7%-tzs9hPM^NX&Xqx2^C~I zy$F2o7JxB4|8iV$2*FOA9T=e8sC^zvkeF31hi3<(eK|R$f6+_ zKLAnH)P*h;EVFY{-KU3*&wcEW<1aYV-(qQx8lml6KL7HTd8?yzpU8o$iq-=~JAUpE z#I&)uxYy%ixX*+P<)ai%ndML15rCFL6J}b_QG-Dibr*)vv(xz+CcGbSV_;T<|0%X?O30bqMjHg_i^XXAF_4j5mD}JsV&~8X@3Yli_ zK2h#>o)0I+OEbP>8ZxR4CIZXep6ES$_E4dT8G0}-6HdP|Jxlh^d6S0y9Mk@&fqo@80~L6x!QCrB>`(50@60?Dt}g`!gg$ znda+<=-Hl~?`+l#Dj_g10R2qKV{z)(I~EH0u<@`EsP`3H+?nQD^@w9h(;tu?Nl@g6 zrLo};GYTD6t{(D8|Gr2YkP5ztm!g)8KS5{tQRJvk$}IosTf3N!ZIEf)hJY&>C&ZQs zXDF&RlZJdTCiFXNzpJaOv^0k2`}4qfe;ldw{Kk4}M%we`4vABL?{QgK)VP4?QknlS zztx=ohTNBtLT0@!mkQ0(_(EVKjf^YYv6(g_*ep}53Lj1Ye{&?{w2ZBaR51}()fcMS^VxdKyz}Cu9rYo+;oVo13{IXN z!)O8L#fxb`^3WF0c|&HTrRCZ0v9!29f7}-!sRgm$DxhL9k3MTo6c(|w}?n~E9l2BqlicU;wUz17tG5-PL{xLvg2^T=>M+nuMUvn*3% zPyy4?R&jdlNd>~j=8F z+N0sz5b}XH7KyLT+DtCzXttgjX2TKKX!EgxQPT2!mL)Kn$yG*C{(`h@ya{(EpJ_D> zzbr`pI-G>nczeI(|L&Hm3Mp}NOta_x=b9pS_XrZrnUQpMUv4P(Ce}KY29$q3A>`wq zb~Hkp`!{zU>Px@ej$J{N0z>M+F1F!sigIC%gS$_z&ECSRj5UN!Ac0X`ih^ETz(5^DJhcc9JplRWf%t5*eM z^Y>VbUJ+i_TF0J5BCfoR3_j5QOOAh~7UrcQ#6cWr8Xf^>_ zYW6d;4VJErX+3ZW027&57iydd0?Y293vK)e1^h|gC?PI(6Tgm!ueqy|hTO@zI^SmEny3pf4c<2glo zc!h{-%q>Vs1(dpjEJ^DDcb9;~FzVq3IIXDMVDUJoC^D%lV~d0gF2w`*DjGN`?GjOG zT?`fX{jWt0cqHYdxt*Vy(vYO4*Q@zIiZ7*DKIZK^{KR6a3$n~s$EN&iind~*u71nn zbYGf6%xYN>sfpc0j>yA3pu5E6qzG+j{9hqU zGYKa5MW0I;XO##lT$-bv4AD0BB@mM<*dZH}T;iF4M`I!l! z{gjUb1B3@F$$NEf?sWRtQY;#BbmqZ!DKoPdCu$xZQF`_m#^ym=3QFih=(cz0)y9hP zEm&B(oJ5|53*S7Vo(syvh$==Okdn~JaBlUtQJS8D;zDF0E@a{e*sgZ~!B`yd?M?7) z+5PESozZ1_RoBzY#Wz5%gV!zY96)fT6b(M?|v}4;4(egwO3#P8>iI;N!oN3<@i+Z<%E(cvCWg#nNjhw29 z-T>#S4QpR8>{rj7(-E9$fZoEnyf4{O5d;>D{zc>|WgkiSiCr_q_R!{?HvBx{h_yP%xa_&>p(v zs%mXSjAAe{XW2Sp<*n$Kee90y{@5KI$6cd<7mY@M9NeVWAEZUi!w|lm%a5$rf(ET3 z!lV-rFQX7o?@RRjm<&cC@#gIYx;)jwH~PvXQq57b8RrahV3#^z2F^U3&kF zm{poWB+tgijwE4b+4<4ZylQeF0cq5rtwa`mDiSMN-#of7L8Yo`oUSZ=^Z?uI`r!=_8WPCuswsI19d0 z?F+gV=H}o00hTz^2-;q8qGY1cQY`6(D$SBi>9GG|0nUrfy=&RQF&}$%Vt?f+ z6`FYcR#SpNunrq39T_S8{9;pYozB63``~?hd#gZKe9pLa3iW-F_^SRlxzLJSLP9&x z%M`Gw%(c>Yk+E}9aTOA;x0&;8N0@99>eTN3I(es?o`s-C2aWB6p*j_1Ak zYuMK8j-De~izm$cDO^mQiP+5J`@m!YD|{Id|fR-ylm_$Kf4l3e@PE=gHz2cQDKtq*Wf~{lvo{(v%zt-eVUT z5KyS~2sY7;8Fxm)dTNFYi!C!1{hm|{L+$f-4Cfk3`bZ=!YHfh*<^($*9?)Mxv_@lN zK{;F$QcGqH%z7(Aj5(+~c&arQH@Jg2!isN#iVdNQ-;KVSlTe%S{ZiM2;K|r4*2GCD zqyI`FR4_P;x@FfZhqo<$Lo(ZpRH-6s8qEE3S|D(QaZgnm^7j`)eXOv*05qr5uXf9m zR5D*3M?vq9cT&FGkUL_j$jl!J-1FJ*2ZAzvNY~_)rR=u)y|eKd*?`GkHknl8hpSfj z(VOADdV=_p0jmKAAMCYiuQB9~iH1GysvY5&n+ip$PI$O96|Ota(A>4%WFhEc^C*K_AY$$?mSvaOtUx7-Tm|hlKu;?vi2nJp;NN@o16`-4f2wQEgG3Tju=D zOzzc>RgVKs9;>kCS{#^TNhXWilpGX(;+lf842oh9QM2{LHqnYIiR>wYiFgEb3{HV0 z&NW;rtAK|(RTobehr$*tt{yE30g%X43Z@A@1*1IbuXQn_h$0yeoIfHl#)r&v2v)Iz zK4dHsgzvIKP5MyCw(uW2{?L#*iBU8ym`kR_Oty@lh}*qMtGj+!^g#nqIRg$=4u zPF6sF{(PW#j|3KT5}s0h!GeCb`GwlA)$qtJZK9-iZUjjz!;{Oeofx9Joj7(qu(o<6 zn`t65KWgT|9W%$AA2uEnE4F^QUB`@HDpp=5R^BRG2+|3rzjy5mH6@P5uL*dFYF8M@ z11@7hOLmAeT+F#+DlRlL@S7!K221 z)Z)i<6hTj9Ou#3n0J}*|8envYAcQ);`yikrEst3HJ>1m_qEmIFOdRizT^Wn@@C;1O zEo!ZWi4I?1YYGaP5Wj3Mw|KdWs6PoteBn3N@^8>%nQXY>bX9xJ%egb#3#h%6MP{LN zrjI%G1J_j~c0n``k#vR}g)ngN0p zX_{vtS_ZqVcV*`y_Hhc4e-InCdS1J*)R+b^1?Df^8k7zq&M+?sKS+`h#(B2{}c@Wj=iS< z!STu7jU679Ukq(;%0uTMX{mRHGwYmXa~k29Ye^bD#ZLO`W6?s6(x?Vo-yB1Qzx68; zmcV?uy2n-dHSNAC`(7uDuy`a9&-omV=gmPB!?23fr*i9Z-rAq{%HlfK_sR`1UybDoN_%ZhQ zn+T+~>;f`OXS#mn>9t}vSwp04Xjd8g!W>DnEg%6NE;$ggsEyh%{wyl7H3NWH6(CmysUA&thLZ?QAv7u@KM-#eE-hVcm?#z7k$ADn zy5K{2DIIVphKpX#fF{+EO->S}ZbSsR<3l&Dqy?Kp1fLpWi_Pbzj*Z7i`h717r~G4_ zp)ehh-RZpA(DWv0t z9eW;fmH3bnIHoydZJaD$Kt4Mek!*uMN%xCd#{-uWQJ5tg z=h99R1@;$@R!A|$>9LfFjT9anPCN-RvV<%w3mc=Ms_dkE`26fT;^%fw*Youj)BDOz z^1*VgxA_gyUK;DAcKghhoOls$ZEF=3q*fHS(aMraa~q4Vy0ZGkrIq@|+PbEO8V1q9sDIwE=S#r65s02$j>pVB6fyhyGy_zI~JA z6~D%{w_RKeV%>B6QV|!z@~bgdweJPEaSg45PIhNQ{MbSL=<0BGcWo)@*Gzs4_N!q? zt_sa@XgZ4bG3LMYeVRKq(U)*+C?g?|XEbDsSJZS8WBB^~uCMzW*uIf2S%@1s=#RO% zxqjX2`1trq*LZzFNlDl%e(|~{ovUv^7S@hslTGtZ%LR5UZMRADqM+nOM|=bu`fxv{ zssC!0^GPbk=MR9Y8t#`pyT%+Ahp`7>&AC@CbvSxD%Q^$8fu$wNQ1z;)Ah6gM<0Bhb zHgjD;KF@b&-Ige?Fds@wD~Wi{LVE{fjKD@103S(Ec!Q;?@Uu`M*Uy>7Hq(XmjdhQ9^81Lm3T>JKqz|H(dV5T8$^QWRbvOY0INcOB9+S`#ZW zjc#%nWP+zXOGZ!NW_hZKh`jzFFC6zg6~kI=kZuByiwf=D1-*cE*?S_s!N|;dn^Ok@ zPJ;)1K%4XlP}C}o&5P!9UkqhFZOe&AtC!D)l#uz!4A{W*w7xV{TGW0Nh95^XPgI$> z>od(1DZF@-KjU{`z1iQNU{C>G_EW;>sALKQ+nCn^$Lzv_ChW2#0q)YuH4oHQ7!M;1vhx%XYOFxhPIU2H|wb_byt zpAF{1us}eJ5zT$JQvv9v6|bFl)E5-EPe_A#H+R0Ir{T?4=+xMP*CC;T34C^LpkVLl zbeI-=Y&0~gbO5bXm3ig+V-B|+{cV@i)keLyvV(UlMi0e7ww>YU-@9yf?Wx8}D=SSf zD);gttCpx>4VIy;HbaOLH#rsBs%rchf5vLJHM*WJc@Ef9F?ej>dVAhiC*_Ap$e9*2 z0pb&!$_lIbE=Avixqj^LPPbqjN4@wvQuRxAYUQe_{z$w8iqIsznO~?YjV6mrOK2Er zEzQi#%+2FK9)CmEP&3QExubut{{I0E%GkJtEp&B1lIC>2!P8Kw*Z;}mwbs{c_g)Sn zlLzhmt#&d8^@b4B9wsP1XhrwzSE}nGJl0Uht>xiC4Tf5kdc4DUBR#&Z_v2#?24xg3 zM_9!>wpk9Vwbo4_VW-k9?Dq2bKbW}X^@@2J;o|3hua+f_Py+JL}WpT`PhmSn0`SoA{mOBL=o{K)EBzi-yAD|IVL(?_l%F%WDf-TH4EVHD#CHmS2rM zRIAxQoHd*(Wi&J(A)4T8T3I?{XV3S<5Ob@|@23%Q-kPk216|;+E_Sk^D`L=0E5@3fl1;fPBArSO6+sGB z3yS1q>qMa-g|GvHP%7nr`S$WV`xs?W36dl2zh}2Xav$WeJXC(-{{%IBe-t+?MkV(W zADGko{%uJtpjq8=xc5{P8bcWd7CtQv@BVa2JA4jPlEwDb*`m7re7U8=T2%?THaFBv z{vri=FD=6Zk`$FmS3H!{3>iX$D^;s65?x=?!d`**zUu0#mO;C9eqpYEF7ba)o={a% zx!AwwQja8QBjJQqu)EgkGU<&d(gD`B+AmZOzRei~WCpAu37`KuwM{r8wVV$syR*)L{zjPR?j3(Mg8o~J~@_UwS4p9wY84nbwDBKj2rJSMS`u%2-N^1K-ns%#23vvaH9nSE-3GsI#_i7 zTk-X0Mo8!RzIb|nJBp>WukKGw zcY7Kb5ZC2l0t8p(RT^eWj`SQvfJu{!9s0F{yyeN zwPi`&<3Ld)&x#^hTv=&jW(JLi|8F|OlAYQ9n_Ff|L-+3_MC-qp&9?phMWeWew33Fi zinF&jALc_8h9qK0UuX(0NN8Fq`G+FtQ{g($TXp&4D?Vvffth@)m9O>viwNGpBl^&r zHU{yeH}z+?`5$CI7aOqqT237~HDwrR85wQO!=h;aEvo;0*xC@$aMw@~!y2wo9^&vA zzKwa;=_*`d=K~|wE;aUm96m1O$QO6G--5x0I$aqmO-Bf)cmi6%zk5Vi z4dspz?%wICS5;N1pThok|NiG-M&4JF_V*pTgocXo`_M`FVkL6x?l_?6ik90VF<@Kl zB#grK{YkrZU+#o4a|6|B{?jI_9Snu+ySAjnqz2Ze*oWJH>pog2L+yY5 z{ofbVkdhi)^;0auY39-_GZIr|F#b0b$aLlhI?aRRkBOEZff|AzdZg9}8qcbFsn-&Jo7 ziLkr7OXBLVEzb(Id|J`+8im0_5NY{6-gd2R6Vs~-R$5U|p?S~vtzLxpgT0xb$#X>L zg`e9*rr)D>LZWSk(T0_Nem`emZtt8F=CFTN;ZPk-MoJtDIu<8zdAU%i*8kyMfF*Mg zK_iSR4*buk>;jS_vj*Y(B0UA(ozT)umEDf(xD-;Gvs=a*MXR^E8HoZnsI;s-h(}y8 zWS<0t;&(l%X1Og5q%qe6Cd)%rFvj|8qjwZk2U7I^*X&+b4ej*{jTSeZ>jsRuuqc&a zlg%0lP`3~GGH;`4q3--dy>Ydoq3h{oZNt+O92RDoXa*F0a_K({vnS!->kDD{x3vh=het-;)v+8CGCPb?i1 zkZ#uG+0cRsF=t0`<6|^K8VVD%wWhfEdpQ}n4s)}j0Np`2#XkcuQS$LXR zKt63s-_5=lJL1!ik4N3gM%`Mg$sDy&e!9@chzEO8K~Ctd&E-y}Z8%b;K{0dnZ*m2W zbe*FAy)M9ZqzCZulvp*gLc#1O>pxu-1^m4phpsuCnMmo#Q7y&!2W>!p5b7?T8o37>AKA(_kW~uk_&-||M` zR|id1I}QJ7PWw~yFmZ3=e%i1Az8tGVMYD(7tY}L1I&;jfDy{Unom#dAJumPbx2n8L4jKx#0l@c@M+Ba0*`4TuO47LiHU7h+0lvr?NRTl9y()_PZW-TN^YcC+-i|o?DOCnGK(A zE-?lwH0t(lTdsYc60#TOm$H64pEuv-MlcX?+HJf%UZ1yL57MetC<_$m`06nc3m@exlQ@ zx$4K!dD*RcOHr%;Q&8o*t(l_ZFkDiUQGZZDq5II2LgI)4M=Iuz3y9%?uCrXTJ&tHX zqKFziC$kEqlWX=`oDLP8yX^-FpU+&iIecDzT!mj1NT={!{0?k?Y`-7CHyP)ysHr*Z zC4MwhQo`tZzz01nAABR%^|n4cZ301}5r*^~>71%-zt-ZZ%BpGEKYo)K|Hxu=(f(*V z_LFa4{{F3?pdbkRRF}QhAN6>)=D7}jyGJ7AbvtW!k9j@4^nE=B9l?j~wU5qb=<1J$ zaVAbPlz9p~m;CANIQIp6HW2YM-dvP_nQVUJJ2y~0cqM+RZaqFKtK>aROc;#DZ!(+C zC0={nzMkOO|FLb#@V)+l2hQ#Gp}oq6xkx^ciRLy_+dL8qs~ZI`icNzYGf8O&i8=dA#A<`ha1ZYP0zt z!(tq%_Twao+71g*QN)T&(Dx6?4>Ie)30>oC4N2etWJ)e(Dv+rs!)pPL4=Au5RODf#y)xU#Q%BK$o| zIlTKWl+RnGJxi}DgP#VDE89$7+~&(FUoYi--?kzcON$3%@LhMYQV4xt#-5&nLx`S^ zGt_jPBd8Z@bX;(c)WL5j0@;C~a$%O0olhrUCO>lclpmD(m^-O(;T#h#;4#e0`*v%W z>-UEE?rZ9r?i_<&?yh!eA8yrjo#)JpG7=5+!~OTm;nPvaUMCSape}C43;f|E^g?cq4Au9vf-%a>$(;Qi@V~%N6mhaOx z-|O?eVLRX((&P1ft@zKh&s(!lcq!#lr#d|EfrownV>{+=$YW!($KmsK4R!}^$Cn;q zkNYz=j~i-s;m$`x;z9C#)K-^MkLS}Sxilx}E^ozvJd?5J!#v`ZCfn@RlPhV)Oy_~a zjBJ z$15rQI+~iEXD#~~juT6|kLOi3YFC|^o@$m7N z1eSikb)0ty6`Wr5QrRrG2dy<&w>=zqt2ob9fVzyIBF!Vu*E^p3pn+o2(}Zg$B3cM) zH2+gE@tVD-iV8(~BamWba2vBc<8wJcD^Ury-qggbd5sKHG1vNRQx2^{3rni$1+Un? zc3)Bvph)Er@mqa-PUpH_MDczo?2iB$_@8XqdS4%xbJfulg^z^)93&il^WoI{O?>va zttp+nno(a)qeR*D_M{~#)91dQ)ywcU>U}RFs!2LCwQ-#d*7|+4XA-B*2EHj0*c-^u zwmYJss_UvT9P~L}F(^&MzXmMcb3TT=9wHUprzSnGJzV1F2P%_EpCDbw=L*oh`GiEv zo@hJ$)WGOxeZH5S6E4R^0YI8`^n^71R<)`9KKQsr+u$S92Ik#y>0DXrs;{67A%Qn$ zU2l{7$%R@MG8#IqTIJ5`^yW6znCCW<_=Etso!{Fd2EaYrn!K_f+4Cf#w&X@ffWIZ+ z_xjl8b6>gka);I%Le+$^hf_nLiQS`0$p1JDUTpFO2co0T2Pw>>@`gOI}Dk5-CM zuBm?B1Ej;NV*%>h$o1Zuaws8DdqX(PVmJUbm@ojiMUk_L%h-=EgVJ)&ADg z5F(u8y_hdJWreNb96x{#s*9qI`(S*V&PVbLPkNa+IIm~X%8~=SEiTu0^8jq1f$`>! zZO^x1ckYgfi)`R5A{&!j@M%5Oxk**YAvZEH2_ycQwye11XgLs#ns#zLJpNOzR(=cA zXVy5;!WKJ)YsOq_LCt%dX&bl&VHMe?+N-Ys1D!`3P7a6nrtNHzLpY~^{gw`SWQJo@ z;lf*kK< z=ku8=XJk6yVD+`AmnSLk`~)fyKq7C?CMz(s?c2;LZpfE^sBs_5?etCcrPnU-YH}| z-u+DHuXiYP{*Ktj=t^NV72;Wi*UkH&;9$=U27`o%_9j z_cMiZ-S<8(hQx*#7=SQo*HNkEB=7Mmiq}Q2v$J#g)FLV&88T^x@47pW>zN)cZc(Ps zJTT_%u`U=U{!xfMYcPe-bBwuZ-QyXFqw^ey3IfMuNq#uJ=a1AM)5mAGF&YnHI|9z! z0dPWed~_-=ccYA5iL%OkH&eZ8TJE@Ze%%j31Wf}acUbnj&0q*x&09_Qcng!*ONSIi<=sz+4l2Gq&zR|gUR%osud#@ z3>RU0%!bH@Wi)=+Z&ZYFh^E*iyYD2PtNPK}<*>-2qnRe9%0{c755I<~41b@KoQB_)1&!2ppM)6_d z80R}?JuK02c%&)Hd@PaoMQ$$Q?!8dfu)S@l@;%OtU<6;r_#zBHjf>%AP5M5anDY0l zDlN(&ABR@8KP}sOU9S{ny5NDOP8)%krjwj|-f==;yWG)u@QGsdOXPu_@`AFi({xaz z7ZBrkEO^>*KCdFC%GC;WOHpMGF+SW-36N@)%lkYf&X}ydT*|}Y9LvN+I4_!+=<`Fb z_IT~&Er1Scn8bun3n;PGsJ>-b(!n1k!t=+(^J?*i;4S1kd^V+-Tb9vSr;9bxhJFx6 z&=f_h%}8X{NY2TXzF*(X51_9`rN-#m`*d-fP!#(|wr4QxMJ}w#F=*=U6_MOkuE26q z@iasT#0Cj+m4!ffTn|fTMYYNL8n>_fU0G2uTgl(;>0ffM9b2^>2BDWwilgIn=vmoi zzg@UG@Cd|i?(CHMI~QRXm&TKP_f_XU=g(Cgcif;CHa=YRx*FK@#^Q=H?MAD|Ch$J^ z^$q2|T(gT`I^Rpw`@F-~XEO)~9}jPdfT647R}G}@-3sW<#^@`M8NT`>&clPslsm6 zevrlezRGvpRbnEO=X`qbt?-J-erQ7z^hR75uKPBk2Gq}(jD0qKg^>On#O3fI=sckk zc%+%Rc#nu(JMivjx*v`^R;oFjG`c3u=1Rvxl`i-k$DxYvCpG?G9dT0E9Q}je{!pYp zS6Z)1Ef*u9K1Cf9gCAhYsWM2gmE~k6XxneDtb91>a_QsdO-ak1SNe`Bf2*e0!kGS$ z#bMBHY_IG*bM5L1(XhgH7-Jsjn_6Aif4!JrQLEIto%DS>_5?jccDc?D#gVY?*GeSpZ)$gxgSdu{qa!8eP>6?>(?&e)2}S6x;pKqdl-EV zyDE_H{|3YC{oq~jTUvFmuQzlvArS{M8LG)-7LtR|ZZR%b!qq~4d9<7=Wi&4f4!=Z} z+I0YK1`yFI?NRez?j!&p0J>xsoP&-0#4Me5>kW{E&G|#Yrd~Ua(hC@omgnKJeO9Ye zD%!d0VD z35Oy@^a(XpKjnER+Nxqj$4skt^^slc5weYqtW>TI7ps(r+~O(1e9YJg%geB$iP-PC zkZM}LMzJXBXv}of*f7@P?JX$0p!=Tp?c{L1-GvR)>~sz6z3&oW`j@rmS~Gj~U1$_# za-09jhkS)~I-mBQ$`Ojqc-Or!mJWt_qXG>yf9zO`7{{y=81g$?Ym52FoDtWQe6X^_ zq~1{>sc&4H6z@;}v-;aj^)Hb&r;<(5a$COb$S;MmnJg}2`g|UD8|kzatRcUP&_CL= zox(<4(E@=JQs8Q}1-d|FPt8|KKcrGiOUpJOXH-#n-;yFNv*T|O-qKn5R?#2vSmr?D zzVcf?PlEZXWzuI4@E`i3ai7?>6ng3pXQH#+3DTW_le&K}QcVA~x@y7a`{vbO#{tP$ zOKo?yOJ$nL=6)4tHJ2}OPlS=6QFP@Hm;Ze3yCJRn)Wa-6{CYr+!pvrppeoPzc$1;l z_i1@sp4WXhNlhk|sZ+Ph2MFi|-X1bk1%j5vL_>a?I^)y`Tps85a%r@i)wJqOZ}nU0 zgqgqhnK*Xb|2%>1!G+6X^8qZZlFMZp%u2O&3diq2f7>4NE95$sh>|hAXM;8fpTlOk z+MFCkzl^fEcQ=(iYH8DO24D%%r0v*CE7O1qSq&I*Y7BUb(0w}s-~+u3dSMyvD@h;O z5sXA#XfDT_`4%96ekF_b#_-|W$Zh(;#eEejRbGWFz}#eTDPGGc!LM(xt*tF98-*3I z>-g?oS)Pw#P09G!F}&`~`M2Mjib{()P<8gaI@a}-wOq55go1P}Z}I^hs>irpl`KNJ zke#eF#n5MF`o6iJ`z-KOb?hBcNGFeFy^a{h2-G+j%R$CQ6bO{`%r6U9*#MXG!49#G zdAtWR$e`0QJ_LWVzN^l_QE=heO^kCB??qM@ev?YM?-H|9Qfa!t)8>6aWYuDINJ34&KUx2Lh&b5CxWz7`~^#UbO)y406y*F<4 zboc*usGHbR(W9|VrOkJr zU&?2xS9?tW8C%v-)0IbQeNb+`xsD1&?r-8feMGJYo!h;d_AGr-)87l7mlZ9J`?_}B z?h-`9^N^0sZ%?kirvP-YvMrB!WMEvw)K7NWM#=Ti=^fBzkG}Wx1A*Vwz)^A&SFZTL zD>SAsG{t)t{af-&UT$Ho4DToRpD3ErzovTT)fUsjzn@ysY~AwGxMD6M;aZBI+E0@r?4Ec-xmgwHsu9vHm8&B7h)!`z`!APzX z;z{0KprHtl%V=l{zz%Et`F46Eac7n+5bz5?cH7T5^dr_&okyj#0$|OHZ3B=|P^bu|wngiz z+cwge?aL;L?Q<`m?~ff?OyAD?tedv)OVC%JFDO!U#`Z=R<eTsyB%>n2(F z124~Ubc}qh1bGibdj(#8p=8-@!K>-mKLUP#^mL`EPB+`@>hQjOUz;k+&I|n3BLC2N z(Gvmel?w^I3pACb&~v8xr0P0B=FVh%qo0@hisp4=Mv24l{YQ2n2*hTeTm5}BrOGd3 zPwNxgzIX{2sFszAEcX-Bja`77{S1H$ts8*paJhZv;_=;xlxMT9@9#IDAYn@6 zxk@E)c^OA8QH_SAslcpNZhGCOE}gR5+J%Xz#9RD2j;?gIu<*kNX_UplC*pm%BbA>C zi!-NhSM>0V+{NX+ci1*f3uEr5L8TFg(a*-ayD!HySZvz63x^mQsuR8@zAcj_J90Ij z${3H7?g)AJWiz`%Md0cK-?)iKXf1Q*B*6pl1@4n6e&1W>2v+MTN#T(>&=w~DgTq6@ zpiFeY0Ifu)UXk3E^M@J(#fUfzemguXUt+R{;jXi~ZVDJzAM;k$uPa@JVd!LZlpMz! z)b5F6(<@J>^9JR8yr|kg~9# z*J<>MuiH^ck&$~3fVsGwuOqd@sIOkFV#7bZSkegW9KTq{)bRR)4@Ks$Qk8cG(hVV-WKjJOGGz zo{ec83g~HXq3c=z#QnIFuvog2j)=W}UF1o+`m*|3hvR}uFNFE7h!*^t94%0BuzV(y@W)NkS-CQL6zO|b}xF0DP_Viq=f5s62 zXRoz7mz7qwJ{F>&B!C7Wst7w@SVH0S?nYans5#azh=%A3+LIA~UVlzFm*q}es@oT= zar?r|CNZ9BuXqK|V3No0F%v?{g;?w5ZliKiQUvZU)JVGarWiwiv$>!v7-V<$tE05D&Rt@bV0(Vs)I+PhK-#f24uEQZ zaL1r0YE*tx3+9X&<#c@)is2MD`Xy5#E`roc<9)nJ($&uB^{)3*4ENC($}KU!`QQKr z(3)P?!NX3uvMeS2#aL~3{YC82Lxx6xEb}Vo0X{htZs-O1MtSO8r)9uo9$Yhshx9}3KWI$Ws}j$YN+@4)VDKqUR2+;1XsrbCQi(csBam6pS>n3WL^5TYjU?E$#pMNCjW z==L`hN|yaVc^+^AmBeeEPKSRWpUFO>=4KXYfM^h*7=SZtDlU@%zp#FfI7CVa4a+mA0xmV_dHC zd&AMtutaV5V{jDy{xC;Nk>PQ2MK<^)#pfn(1?cm0yB_BdvgraNj;fXOp4KbPu>kySuwXpt#=soqNySzmh+B zlB~66=9QV1>dc4gb5)o!H?Rk=v7inQ^w}lA=&UZB;WcK{iCpbl#gP*s5B;qw>MBxzJ_?(I}qc(udRRfw`>H=6%6VKIYr zyk&7dGc=UY+~Ocp>1OKh3-_kinQe@o-JAPimMfFizKx}&p>(j-VkHr)AzW9kIseNY z+`p%d`_3@oB0| zow-rveZI;&fjO`pvJ)Zvyv!_g9f!$!Y3^{{k<5SjH0-(YD(6D5C+&!pO*dfE^nP@q z_xc}q+V!+Q3AB3x!kK)ssFmsgz257Sjc{}SY|X^v|C|A}N^M@FfylSlXCmjhNPIo7 zlGauawVeSLu^#5;^Jo%e{8PVYbo^6WVtlK&FYXL7^s$*(evl@a>aqI{>`hJVM-*1# zjOHINy*x4T5%3|Xx_Cc15n`x$ZR3HAeJwFEIzso;{gD;l%WJCHh_2O%$Zo0Li1{IA zZ5UT6o1q0VOSiuh8@_C-5C;;@Ro!178TJj>zWF9DuA6AuL3X|-2@wcCL_^878@7iD z6W*q#(Ix&*2gjPOY2JSp_BtS$dq|EERUar62>gm1g(Fpf;ge9BS4Eg{yD!@a~EQ?AztIh+u;n=~S|e z6zZUhH^)cFxF-X(#lE<#YWHwysdmuHF+xv3zscvKtMdWU8)Lm(Yx2@;4`d0gYW<$Y z&L{NSWX52DoEyqok)R$+y6~N^~j#csJN1%}H2f+paoD+yEY|)%DVT z8lV}$cbfdJXD<`)ZTn5{_x#@9Ci=6%3e_FGCWnDL zDbGD={OsBlzD*1Kb|((f*p7F-jtz*)|=y1pYHB&Ulyy@+ZJ~ToX!_?G#mc}4?G3) z-G6q0ZW}`vuJ$3}_Q1u(o#)g^Raa5>gd!)e&1;p$3^gYj(VQ)-EGaEtsWWq(Fp4`` zXyA9u((wFcJ(F`_^!DLh%I4e;HgRY2*Wp?BR9^4h*>auDYI5@2c2sCn`_2&U?3a^M zJZ2Bp8r$p$O^2+g6Rgk;lwbb-9caY!L^spUda~{j9 z4G>XQd{{kWqtHV zHUns*5uL8m>w0~@t80P!ipM$635$rnRBwU(7VoHw_KC;0i+gL?M`&;}YPL{%agZ1s zdi;{*`ntE+^@iS=x^MpgF|fu+>`Ld|iG?fkQ2KF)BXqvQ4?sGHXB;;=%q zcd}auriI5*Sk#AwW82kaN0rXosH!a2jTT2;cc9O@3^ly57mI3e>stV@{5BN_#P$zZ zy8}@+=H?2hce8(FpXZsqM~0p^?t30>JJa6A7Tiiv03d5!ZY#4%rI2a%Y5Gkc!5Gln z#xOA%J#aZjwrolFP!1Yc;PdkHO`a=>p4zB$y8;ZoFX03>d04x4A}7<4B*z@-*l={1 zulrh>XWatiqSg0kWA+^K^`4TvQiPgnm9ooA65~cnO*#8DHQ`&5Qc#5W)3qvxgBtfP zMk+v!kOt*9|6I0)Z2uJ1KE7}bWkuhQ(38+9JFE)nhrnfMpQ)ZSuw;|AAo|LreHPv? zdtb=#kaHj?)Ss!bFG~B!`Xj_9sX61-4a5DO!|#h&y9>$i{)M>cU|1JE4h`&>qVF2 zIS=_61^l+Jca=IV732v;#Lm-&i?v^T{+00qmZba+yT8aqnRYW4kpj05y^e<`!&MPX zZ4P@wy?6|%?@PV+W!)vpe{#SZ{pjeouIu*F$_W|@0EN8+s2xLe+K^e2)~b= zrMzbm+g9mzAfOZ1y3cwL@_7Hn5(|z}$Vpu4vb}6?sXF5>@@{oHcGycK0QwgvY}^f| zIl0f5I&FHsIA#=#u9mr08AbDl#U186dsbwInf^KMOOUIoEYWNWf}%5yM8=*BPc6o9WK0Q*dzS-1b*V72(IW{YI%7y6RnSURUwp2*p1-DN_~ z1CR2X6@J(4+^kR4S7zbYo7p85IUSwXx=8^mov~{TaU3i}e4wO6{oZ2(gDL!?|HP=S z<9rT?$h`83`lgIc*30p4UoU@+YZm3W08mrUMe~9JuB!76q%C_47yJC3ipzQ}(ZLz4 z)6tf0=trw-j>kWpsdjE1;ntQrU-d3EEhcZBcs-x^;eYmah-+47ZT?HydlrA6`^2)@ zL^iwG(NvyUHfzMIBak_488l+=(<-i1aBIt(aR0-l!l}@jDSV+{%@gD5SSr+5(01FX z;pXraBJDcfumCr%`$8YG>kiC)CQ7pvex`b0!Y`dNyl*2vua_S)NpZ&us5DX< z*h+v72*a+KWV}wg#8sGKN-Zf#4F52bL^l2VLm|h>zxLGT*PLBzc{OYApFn#luTq|% z!Z~U3jrxAo{-0+1vA8ehDSYnIL1l@5^e$sy5+&cXJZ5TAwaN#G%`qA7*Yu9MTb_?(O{3*c#YWMWD;1tP zHs_5t8Ka{k&*+~x?6$T_2?jf#Mx@ukD=kJ6!1Nj|egYQ#sq}gJ$0%#);2@S-i`}Q@ z#E;X8fQ0orS0wZ=smkjS@NLuz6Azif3B3m~m!w^Uh6doM>wxFScB4&N~$I9gGX_)Sx@wmIGtSws5QNgb6dT+py_ zVbF|2a^%`JZo0Ni2JJJAtWZUKW;w0&f?|9CQwt(i%Y7!vcZ1QdE!=?(Q>O9Hn9sCL zeM&Qqbh--st5iP5$HatK)143RXJB}J%E20!NrterD<_&`N+R}zG;HOmZDIR(eEyqw z9`_#nKlt@oOOlzNum^G2Nmg9VND>NWC236m%+HK%ax~YKtuPSFN*-Ei=&9Pi5RYI1n@ZT~Z^&OI49u&BMy-ZX6h_Qd(Bga&~QH zXtaBMjk*Gw_Mop@hM~ptW%E}Onm6X!belya%Wyb5+UC~v3{zp!R&n`PL89B}HQ&>8 zAN{0zlaiT<(j*qZk!&wzd6KVWtS1D%UMIG&znR%DbNGi6j>o{~dBRhoS?dIEto-9g z@TQFQN6T!Z%yA@

  • ^BsfkJdn}_E ztl(8F1N;6FbeDwq{ZaJB=20bu*PVlvHcu3D0J_Ufd&lC4IWad|qD0Y+)WiV``S%W1 z7zK)aexSkUAAKdqe#p}-gtiBap~|dea_|tgJaVW^@BYAk|;i{!j`{C^uSSO0Fdoi0^w**v6$0$hAm3!nBRTFa_2!6?ldAW_yd&S_GFg@IJ zsShDe3i9(a04cT@^R8tJ>m?*6)*I|LhL@JQV$L!&*fA`v%{Av5D_R^IiWcKAf34AA zz{b8Ch`2S*DIYsaO8K6&*zN@~XWLlU6%TXM>~mMqBE3h|OQt2#EdvH9`mn?bpugF!Ez2IWklI4DPnGcKdj$!fN45 zD^@PW>B=++bkK`NzE?f`E++S-HGvW*4gOBl{<1S0+pp8&LGS~x6~g$IIzOcU7pjF2g@e2$)z~S$a;($MQ*8PS>VLPzW&ni z68$(_3Zp<-^A;9n|6u09hhl2pupA$s08MWFDerGL@UUR^P_Ljl;>vIleERf>^6T5z zz9$LJf+o!HBUMb8f1J~{(1$vc79;t5ZI1fHbbIA}@bJ0Dw3I0*!a>%Fk=P^sNu;Pz?$$VIwWQ#DwH00Su)K{3i;0+$m3 z;fahYAv69@S=U~z)t-Z`{Nj~S2sA8guXS#-+cz&!LkZ8-qa&85$Vg+e!#{;)clXoP z#IYtVFX5+bY_(ev0vt&73e+mtYVzgSCgz3wDIE69Xofg6f5z7ieCavP(7kNb@`GU{R6`on1H*t1$q@hszm7Tw+ zt&D=5{mc8BFVW@je{1pI!=nDd_<1b=`jnR}@)0z9qw8##j($BLRFv}fb(%Z6b! z$_;zrQwVd)_9! z+3vlA5B|j#L9@fWt8GA>qjxy6GYq((f;t_JuglA=$zQOsu@Udi5$cdCe2}LOIj5g< zf4)(*uF;6#aJ*5?al;qC$f1qI(}bVZ8LE)rdEf3`^%m|YC8>_`^MU@Mplq(Zw{8_0 zdwBBIIbJKx&N|8En4Y&qS8g-HVT z_n@G_KJn&?a_#E{KSAS=Fl0|_Cdv7K}f&#Eu2ylA9T6I#N2=kCU%5hA) z`f`1Y`SepW0XjVq7NQWx98ycWH^>{W)ZeRk_<5&WlpEUodjzrr&DW(A z#sNH|o|+CrB~)%oW9Ay_+@CxX)5+77b+C*fvKZ`>tGt=fy4@}?SSBU-NQFDH#l~i) zs;X*o4%jbyJ>%5h4Zghv_`IVsthtRM&PPZF{B@ck{g#%Zi7$9qZ8_c=SWvLCa?;}m z{m^J^YU;oIoiv+#`(Ae@n!YOG zl)fx^I1r`#@enIdNe$~u#LtM<70cV_)y+**yDjFj@dGCwQs{pVMwoxowS*yxKZ|ef z?;oFT_i*;|L`9vk&@eIg_713u3QAog!op}ZRhZlaIOolB4FdziSNGYp80?y$=ZBT; z-jb%{$at)kvg|%w)5ls#N>BQd6sz!S&RPGrF2U~T2T0v4O+7rgz?(<(fOcvqJC82x z4tS+=%U9 zYa}h)TckJ4q!|-@s1_P2xxkW27p2j4F#&HbLj}$2YKhq!yH^lGW>Om5-ciqoIEft3 z3;4%L&x=Q}>5r#%U^L?71=yj`=r8miw1nZrQ2d@Bs@WuO)C*r65{}QXR&DXT9iUR9?LO|Akf3~qy<1Tsgq(B}S*|4n zVJ+tK@Zj_GZ2K=9uJ!&6GQlfD$0G(GF&6}LrwKvw5Hn>~IqMxWF)&CC z@B&@RYPbaW%nO(p7zG6dx*p4_%v}$JVSNqy-;mcxqK#HoL1;do*~qVu!(z{U2_vM3kx;LW_gDSV#v1wGA`=w?94vPY%eXO-*orPL2Ya9?;lJ_`h80U z?hu?AHrLx+UvAVlSl$UYF->r{%bncTzv=`M)&D#PEXD!obyF-XEMvl-_BiuyGdp$V z#Pe?@6c72Y)VY?zbxi^)d_=z|kY**U8Sb_HCZ_u=@g=vKZcYonMz}p}R~CqkwID{; zg-@Yt`%TO7;T=97l^EPdZ5@eUI=1ebrw8{PGTahE9B2Qz^OxLUqWz6|p%Fm%<>}$U z%*E-RLjvMTVXCN&kavf`%0wL7u)6w_>OX2~(Wh zFyuem`jJgHqDXar6}jL88ADKdhIcPTMf)x5vIKr*_7{}Zy#|?)bZrk{>Uz(6s#UGe zO>cne^$nDi!bkU??u{^I+zR=cpIS3$2(cx@hLOW0p$h4bd~QhY-MMFJguq2lOvFAn ze!wYuoIE>gm*04!dYsI@h(64I!v`uBB<|AWTsOT-VvjP}{U2)J3n!C#wud-m+z_Mu ztDT^k+VHu#8z6{MpMQdl7If*ly7v6#wK-!_c8wZN8W5kFn3!fdI*9|Em zqeZ>lj^#3Wg-j{Pa=zT|OD9*-(b4%)L?tG2{7i1vp7=B1;oekNBtOvKPeom5z3!~( zqsL6R(T%$6-r3exM@;2E^EGnz7B3)&@+5(KHPYIySqmuc{t5sXSX;xMtrvy zUfTd)=8!x*i|OC7gn(rZv>-Js8h1uE_OW?@8o|V%`vD&x6a5sQX)Y=nEnqpoRTo>& zq(3I`-?l6Hc!qU}WZU8S#+GL>nQKldCfjs|yOv)N%y(t#XuUqD4&VJR`QUmRj(^GR^zSh)(J$0!goUGJ?1qdRv`7bUf;$e3FT8(P8+AKLx@2KE{MOzZYT z*)y5UFO(Lcpwu*&QP>`M;aZBJ1+_By;ZQ*2Lf8f>MphH7L`-i^KBJ;8@#5!besEudRDSGz}3u;1ovcx|kuGSb!6%V{?O``-B zb%?$8A2q#QSSE92SwKV16*SwqoSiDqhg}uWtOs-nnri8 zn+2!JNuTU8S7ZOz7og+u{^FB8Haz_H_Wt(ph_4$@&xkjA>GNFXY_(YLT;Juqr7t5h zmduzjJF7(Evoj3zcyZf>1O^8B2tMh%xGdzny`*%#oL^QS%;r4xo##BxwVa&Dd!5t~ z#~e;DdkJwf9|FBXetI2{8Tx4C!R(Ib4Iwuqag6-Re_8z8I0&L9{~ELWSBU_x|Mc{M z1K&T?h;h?@c}@4L9qlAAeM?h`Ff!j@9Gt8fC^vDYyGO+*Jn|#_ikHK6-@eW~LeqE_ z+T||bN=fLZLpg~LO((l4$To(A)%q9>{B^ixvK-gAwj zSEx6nRa@@i%I71-Ve#PiAB8MqaHNxno(4^$@S@)hl^kTOt>x8e07yyOO=e~7)eg31 z*gK{a_x@&YE6*)0hHH-$vLN*rRE7$qM^ZyWLn4fj4SK#*4fMyy?;V1I7&Nn>e>mI6 z&$&3ach#NDJ5V;qco8;q=pjGOSZk)lkxgMWOjUWMzfy?2eE01nPQ!2qBj9YcjVX3v zmt-$`=D4N!b#4fJr)UdN)|8}?P~ZoyUg0i>|N3+BHBi>i+{ViDAno022b)Bv!2zC9 zMSr|UEF}AJ((w{Jd8_xlk^-+78dmdql`YJv-Hw!x?^h3}1+`7G9dBARYEh^8{Nyo= zJUAdhLrwkhqp9Pty9lq`i+kG9333a{4KYnRBSg`*9r*S7`k@+qeSN?I+@6Y7UNn^< zVH<<9ZcHttS1SVUH@_C~xje>}(78}ROC(5JhF~+~!4xqq^FF?U1?7rhSyuZbUYZsZM2leeLiy7{H2LxK5G(J_G8(qSLV2 zzrDX@tLXp0Ff+q8LZ1TUhHvE6b-yc~)-1*sp(UVtL8n9#da)pSEUOCWKMmwxzJI4G z?YtD#T2*-h##|gf%gBVAq65iqY+`~njq$6X+4wOT34!1%1qB7_EqJ{rW~uJq^K%iO z;r*fBM~!G~A1(c(A~q9DT*bm1nylPeLb@4v_f28%%wj0ATss}8U)n}l)MdibHJ;_l z5I4BvuTI3OCqeoKH5#RK^ak52R#l)T(*|vy|SQ^cXH{$1`rifymQT=T|{s|E`4a&##jO(I$p z9!a-a|KXtzHNNGK{D}7fCH{)EXzU>~wI~tG9(xcifl0tvr>WHV@PhUs&@Y2eLmX3+Gs{G_Q zYq^s-MRK~FjJt`2p8gQC4+3?;O*d-7%Tcsa2=zer%~5ZSnPku2z*vlKdhGt)o)Ea6^) z>Ge=ltxDttYwkJS`T|vt!JEHxBN}`7Ltx?YK?rV% znzGw4tCZ(+Ce^LuxmeBo{H*PBEdAHa1;3lDEC1E9P z;hrQP@F_tPfkK3j;1+y)sJN-_iFXT*7=i8;fEv6i+}{U=Fz962?Cu(wO=gG3$d3)v zLSZ4chia)ba_F$@7*$lpox(Dls@Y1HNY!gukjOQlWGYv{W3q;zxO^ge(iX8FfLoRg zUc-5lPjtl-Pm}=pdWHMa;ubmPo|`+od1S=2-(Z911I`E6N#rg(U z85tf!L+i_q$s4t^jE8u(rw?*Cr}mqshEvyNgd4QK=$$PJ!a3>e#ppOgMEm$)$~W^7tQ+ z-_7EY=#1 zy`P)U{@1*It!VdGTUU_^Wed{NQSj%_!HuC?cIwi^gp0XUaBt+PNSLP8sx61FXEW$XK(^z`KE=?iwF*nUGz?G`j$TuwlvZcd7^_rJ1u`Bf<47{4XbvDl-)(9cWbPl!{=_POv_=KGG)uz(wFQ zOUlUk#Z*=`6*WH9x0{c}cpc#5P%MV`6-?K}4FtDlUr72B^(;h&MSUUu%)q1xhZv&3 z0*t^4^%BLw!8_c;qokz7K*wz0;=XfoEspv8g@lob>Go&MfrwYa>{cJ<(n<5M{b4EAbh!Nnc?rD}UY;ThlH~g?jW(z!96G&R%E1UsYfGV7! z-#uE@tj?j)pnDA)6IE3T&5WI-8c9y|KJ}@=W?m(V}JMem*luu;E3=1%t3AWv26TjCMQ}>?b9u zO9|Hdu^+!*UF~#CH+SFBfEb7-+)3SS)`<<({wf_gBC=n!(@XLluJm(mB@MOYv~iRq zOQ*v%$>1!et>_nwUj`%wJF=$r`wx54RUwDJNuYk*bs-F6l6r`X2Ox#Y34TJxB^z>|hd3zfu+6#da49hA?F49Fi)l!V4$WfTP_npGS zleb}Lcfa})PR*G&iu{J|@VdD%$z_7wM^kAU=TacC3{dPsR+NX-& z2V><{3NJ}>YyrBf1n1*NY}5!TQu`SNJyf%GUU&H6VcI@(t4585@=dful|!q^Z{FlI z+`{}cRdpTcQt-TVTuhb6hG0*vy4hn=PCFQ{yvEk9F5BsR2lEla&x8QqzH;s&+K4dW zl^IPcaOLPYJ<~W5&DBCp&p&u>|J;(5RJPE2{C5~zqLqI&FOHsy@%NX>?rlPTfaG?$ zf*Ae?M%WJe#?OyDg_g6F?)=5mvvo@y$RMcF_`MS7J}?vKK>Et9L7Sk<_`+Np2=2!u4#dsL!_)(HIS{uN@kfoPSx zyuxY_a_D!C5f*iGbF+ZAbmnYtkBo|+#I4&71%tpb%0yRYUS8$+(_U`Ol)YZJC*dJo zjJt~U|Q*+5dP#J6|DTSDV)pGR$ABH@eZG&2~ zugA9Rsv@(tr>QyA{ofkR!QMP2QHDqmNpMxe^Wz9zWFZ1nQpH;u8|WN;&%c z;R}l|`N8EaEv#f1ci;$%%@nC8A-lXv1k2#H7Z@O>F{;HoD@aCDP+2s7rMUB2y3Yy_ zv>2jr$;KZ%-{mnvVVTIzFG(24Tt$7ah%qGgeV0hI zHf`(uyRjf{l5`R3WLQMFC_1tT6!F%9U^^m&Y0EAZY7+gPQ^)Q6wB-92?Al}y_LRmT z?+V`v*fjTX3T{W~3?pO~e%rnU{^ulVvMHTn*M`UG7SQn3dEL(sjJTei4@$&0R`7WU zxcs%={#a6!{djBZgK5Rob`w9!v^>AE@zx!HSwu-=yHF%-yWR&{|i*fvi9AeA;!$-$8!djPQC@dUR=5J+QIN@2ktjGOy7!O1M4yvD}v^pz9GuVF_JeOFhEOkhfC zjs#*f20lKsl#~=f&p%@#2E4LS6a9w(HL^A?5?T(G9;vM_$dbcR=+4-?yLMbYH|qh^ zb>H8@cOUDDVtVai>XirkX+$PFb9TqM_Cb|(O7);_#y1vNVwT)~)I zpPY$|(&bThyP`|2KQwzU$A)qDeEZ#0LyH%26e#6HbLR4(#ad(EsrEcDFd#NcOZiTU z2rF!R=?Pn0i9Q3hXE;zQ@j*5e-}JAZAHqTa;@yXB!q0p)2LGbl{mI>WAWB(GMY10O zh00<+$6yXLCCb6aCN6U_m5yn_?k*bKuz+0N5FA;r#@IPw08}!AAXH7v>V?nHXg#?A z3^uQ%vgN3s|FzVxbPhm z1yv@YS~rq4?QXo54>}D^S>QwD0HLzFLq6z%P`8{v)_w{9Z0FOk;4ggQ#ACR-Ba(Ba z0|Y9u8IY}LK*kH>3@Gc!aKJKjL8<|$BFQwY&Z%^1#mo@7tyi+u6_5szr#^gZz&o|+ zg`PttS(EoPfb!giE5v7F`UbiXh|i3}%HLOjb1VI{xgm|aJLLGh|ZM8ZCx++{wZjSS|x06Pb%}h<|U0nno zSVjbLlH;$Vs=&B_q&vDP0zbn!{R()2U!$45-n3l7uV9jwp|5Iu;8$2i8=IiV$IaI} zZs8k0EfSB19Om{LHvQDqP@dfc>|xP%8AA^#y=UEkw2Ug(rnc*)SYdJZk?O!x_F26Nac(P|a*(JzXok>vIH@gHBp zQ%#Wo1U&Rt>>c7J;4*v-BabSHV-<-$^AWPRZTusftM_<$X-Pv#iH%HU3mAXvg1f*& z84TMUoHHGkH|+FMgx>9%SWxo z_^4p`?>dn%dDn=7&(U|~8i^69R;N3NF{XWZ?{)T++a?6&{UYzR)x*#%q^j&J27Xi9 z3FJlh(N%7vrSQ>EOL}%`>8GCUgf%T^8$pf_M+r>iaYPtd2Rmu6?@7iT#G%cWI#U>88I z8dg-*L)*%Jv)f}sL%!s&?umWI#)e$q{OoxX;+6CE5#q|st zWexGM3!jZZRx95;w+WX}X=G+lG4FaXOXaVXBHg}4axd7f1wVE6B0Akp&7nAfb2R7g zw~^$oCw_*5sAxeF1;tHm5{vi81xQ)w4zcBz|1f)Y4{R1*#&(+BmhLs#Yiq+YLUX9) zq~xZpT3RhH*~JQ*Y@SVkL6gqToUK**y=e_7eb%gP1q*?)<;Tw$7x1YpS}h*0!BMMr zn~Q>h9S+F#;LX^{PFtv$CG3yeLZmWN54oY15*X)dI?Vlzit}X?yE=akue9VD^DRfI zk%0VPP=GEQ;h#hCM*0V?j+`q0I(^*A+li=#~yD^4acB zxqO@j3wD?{LZaH<_>N0s6I8c!VNG(EGeWotc65@j9gw(o6Clsq){Poaf6bubSH0 ze85f1&JK{H;_Qj-)Rs9PKd7W-BYw@Mg8Is<$kWTF;9dwd{QEn6!)K%a_*VxOS#EUz z-`OiFBG*^`E$J^_O-s=eKXFf&9)og%{uVoR<5Ak{o=szAl*9}lcPLBh1r)x%3`3#Q z4x;9aQj=p!iz27EHTt5KMObEhWU*$KQ!Fd<%h_28DQQ{i!L@x^DYAtpA^F~-gp;VH zo;VQK?T*1mgrBD6>(%|+cHIO1#!v;4@zu^BACl!U^lKU2rg0<17X#HNO1|TdFh&n| zOZJKiicWbb{6aU%TRL|!9}W+TN?P0#Z3)7bt<}bf_pzK#qIzG73$U;tn1Vo~r$m$` zX9;*0=&>!7`G|FRD^CeAL2pnbXyN{m_X+;fK$`yfi?Oce8^O@q#pW|ePEHr(fnW_b zR~7I+V~J|ZB`9cx|l75>ttVx$O{pjfJX8yCRp*Zb}L%4>Oi6rRs>kNpzMIu1U7-Tg7RL{2= z0VhGPUw;r$A3qy<-GFG4iG$_l9&7$D3rf}$XSuofnptrX*5_6#En$={z{HoV zE3t>^S|TnDz5WWy`GH8(?Pn)ts#=c^5@b;j0^Q6L2dRSyH$jznVoMGF9VMoFK#f`5 z?|JX>5wttSvmdI0sxA7UeV?6yewrn>`2Ko_Ssk06NthEmHN9Mdv3DQU)>*MiqPy`d z1i{iVah4qie9ycZp%GzUav=B;seWXCw|f!4x2Zm-Y*pkgOB`n8Z^FRFFD@o)V=GV< zU>dBA{Kr;H>RbN$1r35_o|})&oUwZWyp@@AnCO_G3N>dD%%5a{`DA%;a4V6Mmntcc z41p(9Lex;HDG`%PBe$=mG%&w?n2;nVS67_NxtqZ4c3C-C@0;hoe~^itgPefQRz+X` zU;@<=ag$b7p$GyYnaD0JrQ%jmadSa^xfHdE7j$}fe1g(mL0UCh5tQY9Vy==BI~3tX z&f$YACJ`xJziE(ZE6;Tj^P>~b8OETvgH$?U(N`))BCf+N5n&`^p7=TnT3TlrsS9)) zNs8ZM+p6~0I&R+xiHOE0Mv|0s%}h*2D`6p24PW}VzgYm+@@^k`jW6TUFZSG5b}j0; zSS30@@$U?I>(dU?8t1V>3t#TLJl&X?gHZE}Rz+}o|G4rJK_s>j-`4&x^2bT{no{nAq2bPJ)>wZCq26S0{Wy zO4Xu*qr3qDxCz8k4Q$5Yd~<*MHzvju;^tw*lk9+udw|7oCLe7aVX*B(#*Ry>HASU*v zvNoNm!#pf2dpi|dk#Xc(Ra5G@sIQJ>gIC9C%S1MF^UXzruA7T9NW#UJtL?BDtHHtk z$Il;Y;t7!tLv^ljwtY9agI>(G>O?X2<|0Fjhr#3sCc$&og?keYX+^{|10)W##*zqR z3UDvX6I?N$Fk-rY7wf@bLhzn{$7EI*mky z$n&EtA-a-YbD$~zWOWn$D3RWKsz9GbO=hPR0YrKKkWaV=?7kt z?~W}b|2fNC^UwcS*AEjlRJLIKQyO)>tEH`N*4RII$OHly8F#j;B3B24pTuhHL#|JC zh{VKTaaE5NxAzF}JUm-p#94M_|41{nnJQ{hQF^;&$H@QuOp+rIA)=+FET`-3?H#vl z9V}n&FdRN{rv^PJXxFyBySqEJWZAF`CDxsqnp&+mVyDuIC{b4Be%OGo2jrpyv%`Zp z-`mdE^knblq(w8+!|s|txQHJ=XD}e~aOjHgAJp!b`IW;r0wzzypRHq?FZxD3KM_4r$vheX%Y_!Hczd@~k(5RxJ zn9!=i<(hFG6!s5f6@-e7jWLH>w4=+Tvxv$~!KmI0WzkZQs9Ls&{S-?#U{Y3WsIAiJ z^f;oAo*D}h1MA^kJUjf{R6jTKSps^B`km7>d+x!bYSW!DKNUk5zEjAAXJTVZN=n9x z2DJZFRFo8DiwZd?3TyMMPA*HVUsEdH$Mb0aSp48Oy5Fkd(^=mR!-e>=UVDwQ1S2c7 zm_=Hy)8#&&9#wxS-3Q)$C;*eF#k;%pWfv&MD{ZFm_jk(g2f*Cu(slv~Lv`8ev`n`P z7yWeJo9-b8Dq#j26l5)sKSU877@~q0BB0_{%RqEV=O+FcU8WBsk-X3l?g0w!ff_Zl zyA7JT`(0p&J4kG~b|_TJ!S>r4>KX}e85HZ8HQ@OX>xQH(_6?1Sa~Oa7K*^1pH5#sh z*I+WyjP_I0CBuLZfDmMKJ&CJ?{VjDg02^1g8(+2s47E8xt6z!rSW?a6dt>4_EnoE| zzmAECArkgTH6|fCx&(+hH{^N*RYph8we_rXAh-m1K|?YJqlrt4L7g^(g?7z?EK5Ew zLu8XO%dT5C+5F&k%j43DRptP(fVa0dMPyCQBT{2!rP<&Dn;D9J zbI-{+kB^rWTlF$TlNp{NBak^T*f;PuxY;hRwXKXOF#M_YvpHU16(+}2WK7hzQGE$d zkz@3~Fm4(66N`WG&5j+)Mvnn~LVBk1X``#EveIKXKIz3F@vNz>tgf@tWk!$J{w#}1 zTD@Yk8uv5vkKhnFVjp!~+RfSdxP$~m##^8w##gQJIKVai?t@U)Fy)*rRAancYQxVQ z@((?v7kYKtB#~V^>TN@pxuWnTT*|5g=@U%A>~S%#>*rbu!2JuT>_M&RiHV#y^dY+s zalHzZjTg;ujS$FI_ps6B2dkO>CqCv1+fLUT75fYbpBXBcHn%9`=I<%G$2N5F zzFZ;$opL?lfk3RGfITpW3HNltHW_D^Uz`;I!_8sso>9v(`AMitXgt<>K&f6dE*}+I zdfFoiXBe!Ehaj7U5NheE%_R+$yU_Y9;vRu2-xea*Lgh*6pLAZn z_{Y!KJOGsuMB^dsRS1n05{&7uGC?duDSi&NacRq+gb zU;xBW#jcp6*=O+#vki5EApsO6ecF=a-@nJyBmJOz@G_7-krP`NV?$Si$^rk=U~?k7 zw1v_CA?hsy+F+t>QQY03xVuBq5Zv8^yA}!1Vkz$K?(S|a4lP>T-JwWvr}XAK_n!CW zZ}MX@nb|XY?Y-9W3QS?>qO6lv;h$PT<&gh2d=nnn?f_dSnN1PYwB~$Op;&(jgt;W1 z!oaZdhXM}V&0;{!oUYYxkE4jW9+FK##ZFIkAb?jRFeqb9gonj2W6vmDB&3Jx>A8p$ z=xKN7D1PV zR)bebQdR_0Cr*u}{mM?H4K#XojgRmWzeM08<6^$3tiWFaLuNqZm^^g?_$Y0}4^<@a z$Y-y%6t*o{PZ%VS-z`yp?Ym%LWvB40a!9ZjECmg>?P48HUngzV`wldfNYTnvrv6ND zj|Bl^Gmo@X=^n3mCU81h_t<{JAeB8I?V;U{B+@dH7O z7rx2m4tM39OlcP-Z*CoOlPmLJP?Q(uv{LViojC%?nb_- zDz|5%$n|gd+aQwijkN-ZfM7UgUzXY}SH9m2rJI#b09!+ka5QhHl;vr^2$|LmuY*x| z!cFUnhvgvSt!0ue49ex#ft3%tJP>haS!v95^Caf!7u`o|!_9Rx^OsH%0Uqkqk ztQ@j|=#(f1CX^Wfu{F=s7V>&Cfu;q4=95+2lLEOSUlF*UkVzwW-EE=mRaQH2t6;!H zvDQ(lD+{UQgH9x>fZ}cgd-cTL!Dpe0jFtGfGPZ`Cue=d{^fls{ zCZi9wVwn_vgZU*<6+If6w-{vyGBR=sfZYKNekc`YTaw$W(q9#hGe(ItVM<}i_r3$D zO9w&MvFXY$5ukUE?$!;M=;x-QQKAZ0Kz5Rc0_y)W!dH<#B=SQ0x_EHfWHveASYm*y zO1&(=zQqM(lcNQjDQFS(s@Emzswk*Sh+32T&dk)}Yj`*iM5Yk=MLrj(j0!YGwC66w(~0#XK()g}*P-&R|%Q(Zyc4 z2MR|A(TopNrrN_fss7*p@&tZg2l{MRsf5nX_QTp6Ln{)=ORe{FDolev)3;_{^igEP1>BW_G06^m#5e4 zAd6o3UQd?}cohn8J08RoU#(z&_&eoE2f6$z&ILdRzIr0J+|*KIfyLyltDA>gM#LZi zx*Y#Aa3syG1j8%gbCUjRu4iq1R<_5~sOv{U5>^w+8lHhNlF}B9%-lU0>mGj>6}Rgr zFR(p<5MZd0L!R-}fMYr`vxkj2bv*VGK}b)71N{hO=P{-&m4MsSf|}*Fm$xZ%Y^3m$ z@DX8Lq9|h7xPYjo3hU-(k;kZm3b??a?WiQ|=nRSI&*&?~q~RwBISn#Y2075}{@+}X z8&^@K?}8OSLPKP%6_gp)FE6gzSNrQrF^dyKl|Egz$)9ZDLk82e22GVVjF%=2i&kSq zbfCu+I;G^5N-Q=iWyXo*Jxl;X5@Q*1sG(+O>F*|XmAyf{8A=R{S=douk@FI2@*N#Y zrra&)qGk(#=g+K=Sak>Jz@~xaR(U%oQ^7bGj*6uJR_WS>0hH*r<6M+#k~8&Ah<>1k{m z1yHieC~9@!MLiTVj_i|tS&?`1CB#O(W%%C4JTmnSRR;kerYj=>JybF{ z#`q<{qU66%SYmIpNg64P&GQ+GtAE-V-f1HRLRndno{grArvqde&?ol)3>x17x$n@K z&12N5qs~A(tIVF&DrXbx3wZKsQONMY1ttZUZ?}DkMt`8x5xV7Us7>4^wn& zpX09%g{2tvbZJ?=*yw_2D6>?wsuI9Ck_mCv0IIRBV&T$%7v=RR1*$VN8m>4eb{8Ds6#>vQ`?fsWUe~-|Bex~n_~cJ5V{t- z;evLk+@m&d{)a-a)QMuuFeC+LX9;i%(4ei0-G4yIT5WT5pp%=76Sg0?RLjyyVKs!m zl<6HKpf)a3m&+8qJ8k&m`&s0XDW0`d=JpoZQ4 zYx^pJ6plk(Xf2CFncy^d0S3=As*^!zZkyi6*Zv-khU1~{u?3`7-Yn05N-HV#ND)^Q zhaMUub6(+q{t?w#Bv!~)q$aCLD_nmn}ySP;+zpXV3_bR+_s)y{wH#F%aww4>IMiYhXS)1 zvSsW(b7KGopuVdyk(Uj1Mj0C(LvW@~CHEu)_wCCYtdrCKyHX4LohLDFF;otqVYnRW z6A?_Se-8ZG22($`6HLV|o#q2it$qXgN0BAbh9GCeL+Q@4ehG{qNsK3a3!I@Bi%bjq zHmyV(mSy}0mZWaQh5HxYMM$-8naTpHd~13(8T6T<%Kevr`~UmZ(BDpzD=bU?zgPfw zV-+6svaHlta`QXnO#)-iG#V<2e|{^R=&~6q>P~<{J`flq9wBVPaA6oPGyY4G8go%g zx!pTL9QTCSfw)37)9XDi>nVu@JX*-IlAbG5v zA5RU&kM=t=oenK=k>a)3>G0qb;8%xfh$Dlml*SLp)DJDStVrR(-iVZXv z!Qpp4qb~iyz>P;Cdwl+;^FTBu&E1tZ9r?aQoMj@6Hqfd^tucK<+C5Ne4+v@NoqdKI zw<+F|U;Lr8dL#@lCzFRsW={uso}lEgsmnP1HL%I2?T?$_vlhkkgA&gFKmZnaDQ~IK zkP55O#Ikh{WQEr?ykm9ZRNkVZrJ*5`p~&pCjHC%5l|?xPGjxGXSvSY+0Xex~yLBqE6%*5JR-0-BZTHcs zId%VhS8^HoE(0E>s}#P z1jsp~!LSO-wj_us^V=ngXU{Jp|1P2>+oYA2u>^@KG7`1KbGJ5_1*G=-YtWD2bE*B% z*i~&uL=q?|`3!}szshX!OW|yoD@Ar2AqAOvx_u;0N(c^;6h}{~u+CQEmj5JdEWshA%||~!qE%XlY9st?Ao#V(lao};hr?k@muj&xlhhy$bc1ZuE2t=7Y*|-6 zWy64$UQ;8H?yJFx>nss(Xs5p$_3eA9-I^k?vLdH`Cv7bci#9~HV?ew60(IbZ#L!T@ z(ES&n;Rf?oyOAUf@2>to(T1)gXE)Fwp}l9M#o6oGE2DIxbMzF|kClqyN>kl@ zf41rgz8q=UJ6|!3TMF&-tSk&wCdRQZaKnnaWa88Gj_NuLUyDtejmyPj>nF+bX<{>% zo(FPe5umns8uYi?7$meGwzORY5{lbzPE=CfH}%TV0kN>wYjV9M?FzrWQkSe0qL?9k zY_WP#)ZQP)1&9>sy3*tWt5YTuH(q=tE5`!SIJfM7#oq)jVYl#79WF&cja$nVTp$j+ z#D?)l8HSo4>W=Rx6C1qy&`p3@EEMHV2~!GC0L^MV6WC+fC9eA9=(^hr-M48FmWu{B)n_` zCpeD0cWKMRc7O1Hhwrbtq`VDOvJgWX5({*(<PsJE&E~~TMNv4+>T-+F= z%S-*-h#GECI1^!Rkzyt`9vk!U`&lJ}fYBu$zaOaQVk0_O8e6C%?gVC2rc5(R%3hLn z*iCBILDA?+JM`Kbt1EifWMLaxLQZku;D#4ulmNxNf0}tAudvh$G<9N0Vf06U+3;u5 zJ7Q&A2CXyz?~2-v$~Tk*)mS?YxV8$G;8?kcxZ^Y$Mq_DDRtZcpM#gI11hj_b<+exQ zxmfmu!VHT-x?o0}$$|wyJ0E!)Ts6;^ z5#R~2!G)a(3a+%(3sk8|{6IZHNm{`>QbhM0Xsxlc&{~i|0$;Z=H*{~8@6bh<2I|>> zWA6Zh`-m1fH=BdB7_pBNHhdj8)_UdYu!UUp3=jZhzC^7=bvhub%2uK)M+hk0p3uyPdtEXgCz$)aFr_VEvn$dJ}z;RQq_3$-XrRRT%gk zkcy5(fwTqkY9|KTy6tQ%3d((^#}5cUC0oc}M=`W;X&%UmNQ`(bbV^q^HFrdJpXBVv z21Hsc6-DA#B>IzmPj*&Iwjr6|NMublS;QY$O*FDGqmL5Q$zIA!*Kq5n2?r^* z5oI;rICdQ}bP)>7Vd-2rn8W9F*M+1aN%YjU4{ErKtszpQ7H2FaKMj)3gFJ!^&M(^O8Xztnb;aH)5caAncDhvb5yVKjVdR3MYK!vs^ZByGL%a+;5jxuwoa?WytS zx1vVLg#u7t`EbqUzT25oCSh}B;M<7e7AxQtVC9zm#sFMcgxT+A1e201I6aeS9YGe< zn`J)&I#3DCB2)DvgJ{rjoK5k)#7%N!-7ahDS#D+SX^~Rh93F$yaP$1>~pLT}FFl-AM5Smj8VImOL!HD#m2ERX_P z+3tj1^@Uu$`w>q7t}Dx4>?@#+py~zS`Dx);vG@drCoH1o^48;tJvv(R41O~}tb&>7 z(>Aig`l%2JxBd&Du(Qn_xc-=wgw1phxoM)6=kNnALL<)f)uZ%-2JG#C^p6Y#glDBF z*aB#dL}P%Evn*qxt}Bs8q9CI4AUXS#H##y-C-^HJzSUS)5rMZpc2J%Z@K;oT>L%5f#4lZ^1&^R%D{BNYNS4^d&@|FyA%?rQFxatYi#F!Q z`Giukd>Fi;1n*1>#4309&_-$uN#L>B;Pa$Y@i;QWnNkS{3N;@ZvcJdFsd(0Gs3a^t z61SN*0nT|G@2Ytup4;hR+^MXyCkI`2*@O-ntMVIK1xf8Je?*EQ;XS*$sB(b|vj^Bg zq#VQ@^OHPN306$tx{$Q9KV~JH4o}@5_L|N9_lIftu-IK0nAG3K2Xx&RJU?2(o`7YC zEy*Uy+GCV$3SizeDH+3bBkD1Zs6a`(#oW(5$LwPq8eCHhH;j%4=_6p}yy`}ILQD84 z6U9O>x{ObckU)d{`idU(XgVr`3T_;md6F_Z9IfyxqKfFx#0xt#)IZwKM~=@qS9euj zM`p)GT4bkiECe$@FVoa9RB^1`(f+EY{UI^i+B;ko`0PB)eUw`MTJ;3&run1WbC7<; z)(vuEvBWs4UV@9>&RL{D z1@u3s7Yi5vRp?+EM@LuKjwp-F&>upfl^4rnt{5ECl92$)&Z>40AbP*MiEDj2gI^3# zS}y3?UCOukE9G{kDLQ#yvO3Rete&zv(ix_2?!jH)eSrig0{UjS8P;IsN&RnYEl$vxOdrynufv0H;&Rj-mEY3cU-o>IhsCcn6s)(%dH2MW4-hZs(r+U(}n0E?GmKt zl0>7GXuT5Bi9eQJI>SnvWCUbKALlJOIneQMVu=PMpGSeeTN&XwK)#hMyaYOm$~pzD z+CvKGK#@`@vGb$erJo>$>?>tQ4#1^TxV4vLj_jOJZCHJ0QJhl-$OJR!6(jn}D3lE(`3L`TXY&V&VSv z=0|zu3qbAnYiYlkn-i@VT7_rK4H~nWj`ML#wKNQkr45Bt_{Y*`@nW%?f!G|G8tAv7 zrk0G2eN<{;w*z@FR2axJyqx`=l%qzVX+P1{yVL*00v6#5Yp&~=4oT^BBH1I!D?(`s z|5eNAf)g2~wIy$&4o@=XVP~(DphyMI+f%k1(pP%xolD&Y%?wG(8hdevL0Ic%QtnCR z#_$BzNQ)J#YGi0DA3|wXn$}Vyix{;|H0tyZ#keRp)>M~)TLDv$JX;+E>SsZK-`^^k z9W)~W15@l9_W|ux_FU8+F0g!iWoaK*L%{{>UzF_`W1uw4?pbO%I*?F2aDR|Q-m;tt z36}>K^?Y|6$?K(Jygu|((r|HTSW3lUG{b7>m}H8z!yW+_Q7*AUpAzw8>IR%E6(V(* z_uzwP5R&i!&G7C=O7;#)XEzwQxL8;YrMprXn79qO?GwGaw%ApI;%t@9DpdnkHznBC zv`NL`>Pi>LPLG@)2vgYt^2_Oih&0@fSb&?2Z+Jt0MCbX+LWl~ z3C~HnF+ysuuXnL9DPGi`ENCapA^%+j!3&RLR)Wl$l;k2~`;xwibFB2^#22qP6?+`A z78EUE*xCm14Gs{@9Jo?!d48m|lm|$t#s0y8iuFgWDi?^m;0s_%IU=z);j_0j!LU^l zQ2MjaK?#OesR!KndT=1f4C?EM8*-TFcj-Xw3^17t)N0pOzSQ;y}S5QeBK~R#5|Jh>Od>6n`QNR z{uzF0K{JdxJH@Aro7=uZ*|RGrT9qejrofFdeuaRaZQlZuM$?=kKY!mfYEwR|u833D zbcLSFJ2wcnfuDz%aDv{16P>=MRiQS~P0dA4jWtnTavokt{?>cO#<#F;B-9q=AQ>?z zmg9;%B~|8S<7f>--0=uMWh=5W#YoNcMD-JzjEm2Yufrv2T?QJnM1IDaH6rN-{qyFc zcl4mHJf4BYQ4SWXE&OyaC&>ojI6H;Bmv0|(dk1xV5gf$$5esY+;GN22J$v7ZN zlwcs5iCD@u>H48O!lO&8p(fi8X2Yus`({K?5@pXZAg#r~3}(Z@KO~1fEMA&VN)?92 z+{ZIjchyw96T{0Q<5%5??P+blWM$Ej>UmnVXC>1@FNb!|LVJnflEgasz%LO$oyBhK zZh(f#{}X{``edRR46)&^e5>2jB zMBwTY4N%}l5={LYX93mspbefGLcT!Xc)4$(?Kg8`3=^7O1}RV&)T;t*6AMiE9P?i5 zBAqtx6&2S#?mF#FBO}_61I_YDrNX!?J!6M0EO=&36|}Ge+p&(c3zg)#sNRM`24m5R z9T}$2fsYfnzRW}1>V@{SvjL{~R_}Ukh+C%K(HV*}MzRgC;r7T*`}Xi9S|lXFh)i`> zW$hHg=sknw{T!6K`>0){KVpC$V7~WxYubGmUd@bd*b_L#c(q3y1Q=V(ubk79*XbGM zC&{Yvr518CFM|Ah^5qmlihQWK>mj@v{asn>Aw)u8H*EX_yGz=vqFsMhp?o|Z?QqdZ z8SB`;A>YP->*RK~eoh9#MsC0fsqskpmgjV+$5Vk`JP5b0gBW5ZdTUfaz7|E~Lt2CV zZ{>l&B&bgZ6f+LP9ubV1D9pp{@PZ9`CW5%26^0L6?jYOSn~gOx1<9NLI7 z*@F5!6C3Q1wuyZv83jK6lA307M!PtQtf$kmPJRXCE8!-TDa2#jGRyJ-Z0D76f2GmF z;ghylugV-D<-L2@5w&GQs)ZC zSJS4W-1eK-M04@QJ4i5^;BugD+RrWKc4kG!s6f9Bz_czwkNh2`H9cAZhqM9?!y&Cv zoG`vS4KK>6vW-6FBd3*eWocWZA1pNEO)|3Vx@``LfT*#}G0~v2Gx?Cr93JYpBoPu! zaO^1{U|R$yeF!R?2T@wTg-fX&=#5w434@%{?G<=!OSFbrtOUQU$R^Rovj@{?A+!}J zvdMPS%FYRCG%u@fJ%;%%N6XE*v68XiwQ3y-)?Wz9CuvQFaYYRXmp>0Y!D-Gxjfr(UphFETme#4m9z?%f(ubyEcaTta zIvP$`^5-i`5leYi#1>XJXRKGi6LG>T;Cw)uSgHu`V;0uzQ@mTTx&+>*tRo6)nZ$2f z@~aKl(O(RE@;*z29jF#Lqb=YuZaFtoZ(6ixMxpJV5X9Fj+yXff-;sqBY~hG;RAG2b zWsDKbxYnm8>e<*DM9e%KFLSs;R>Qc0%zv7NkHMG)b*_YFu8_PwJ%tWo*;1KKa?D&c z6+NY@^!81%zB03!`UTQ}`qy}~aqZ8-Wy<<_?2VHdILDluvSvguwTJK8f|1vZ5i*AO z0OyocpJ(}hU|@jUd>;vXL{x9pPgV%{$ZSi5g~95Q|4M-GY7xlla0r~iNG0BcNYPA~ zcIdV#fwl#1zSKA5rfA54IqVldbE8xD452fHAmu z+prZgC94d6tC!)>Ln6}TLh_RE`_4Q{o?5UdyjEbuIDKOq>eLrmYmsX=m(=qt1d(fM zyub_!{2ggr8BKS!-Qqz-*|N4$!uG&$@m=ArdzKujC7b;v`6Y&pml?|a{yKF^+(?qh z{+fHljhEbom+tC&LR$Wr8-3sNr7;NH~d(3&3UTsr)Y!Vhw)7pbb;g?!p|VNkow zr@e;&Nn6I8qe&Ix7an&Pk7QnjkG~O+8WT?OsI^gXDdMWPQZYS#`PQqhp{&U-ROQ_U zC=`*QcVTuE43qv#LD7@ai@mcHK}VIG9F7lf|F?^V`erm@q;6W7jtWc73pJs5Q;Uf> zp0d^j64I-_p9XjLPTQYBh5D7$eT4PpTA=8=cbhDK&r?O7P{oNtDA$$dCz2D2^e}wg?*I1nG(o(>?3zM=qijTT&ZL zG<4Fk^b&J(ajEcy>m)3`VBN>xcVG$RLG>3(R{=ZWrzFU~aZ63OE1=L*f}OsQzVnV6 zt0_*_)a*Q~g?To!6@zGq(({l_oz2mtP@wn&!eOq9zEpM@Sk>KK*Va}N>{vQ>0lYL& z|B$-A3iWX9gu4JU|WwG6Oq+yW~;Ot-% z{`C#4dR<%{yYY6=8mmCo61SfBx5tX&(-860L$1)8h}cQH`Gq_@AL{_-F1K3$wM0{% zVsbIFrww7Cqif=-r~ z5$vZ+8j#m`?F^=WFxdoM^#ea>QZb!p!3A{^#H>2TNybJCAT_fp3s$7os*JO?_vd}j zF(y{WD1i9QAR>+n!;Z|Wb$S*36C;6ESwgmZuOU=WddTKA9rC~11v=`N{v+F47^jba zLQ6gUJW@6R$xxCQsMi=r{JWVSQZJIZU}9z_H?Npj*JIRx9>z~;+2assU9N<@`ps60 zk9S1wIG7DvI?(**oB4=m2{saq+M{@EZP|s2Dqs3y zl>1C0eH@M@MNCmZ9k0xdzdqxK$|^w&BF7bSF%##K=Y&9-zNr=#@;db(>@gx&pU-<0 zam~m%WjtTkCS?YC=PVxkz@c0d;)csWNDFsyBtE?&gaA? zQw7`w5XX~D_!vSGNWYVPVuU>4bh(7hMSL}r+uAt z!_q~-w>1ai!lfJYE?E$I9SD!GjzOynF?mA?#AXRW2d;GU*&rjD{&_v%`JldpUx$EEc40Lo#&RUCnn`q1(V zw-RJFPOm;!46|`-L5bMj7uL)I-nNV0&Db{3^n%Xi4svW-BYN){x6WDl3Yg{aHSS#8 zlnE)cz7Ljn^wd48c?H-fo}XPVKIcJ2T8_Wax(ciPA%z@Z&8)BrUjV?i#Ca0Yt?!a1 zSXMctXuW&aGB@R!iTcr`%`Bu{*BKk>CK|YGm!Rei<7TmTf5==8_Luw5g=c)IM5zeq z5v8)~(dY~Ba9`Ba6hp!Wf&iVW%IfN>)8~JjBakecrH%dl)m(x$0b8qEj(6kWDhCHB z56Oyd$i(ELvV&X7#T2$Xt$s?dVxANq{{#L9WZ}ZuS=>#RC`H8?z}D89m<1JbQIJo^ zBs{s_Bh0y}f{(p@zjLz=yGdG7(mYg|-WC<`SIyrwj?Nb~+qZD_^i30m1|Frx*%;{t zfZ;(^n|Fj;KF=oW0aVgc(JEgQ-t8yFDnamGjHD~!{@gjjzakDxLeiSJsqIbJXE~^B zM-wEq=eL{Hg&NPGmCUjIep8Q#Saq=SQoeP%R-gT?KL4Euhejc`qs}cn!K=r#yv9e4#8u?hxkl|TS1UkDV-Uo?k82U#7Q(g@) z{SyAP@)oMtgDQ`tpNVDkz4rE6mbT1c`@QAm4-XS!r*B9v!5Qm@WROzZ;S3#XuF7#1 zc1iH}{cqJbhg|`$WBI$1^c{@2+3}4=!KCSlMS8=i{BHF8x&1ItQTH-9ibJewro|IA zD*_uY%;@4=Hsb?zR2hz4_eS5F%a&aGR@TAwHErcijUO#r^_1lNC1pQr>Z-XYinPXj z>a2GhxY-E!XZ`q8m}1Y7WY5up{5+qff#Q>_4WG3o2E-D^BRTjhWTVUFp?eMOY(@Dq zFdC1@+O_D&v1efpM*S12GcnRzWnU*)*;f5Mr*J8|cw@?3yS#jaSl>EB$MzUX{<&Bi zZ?+Ns_w;+KiH5a2j%HVvsR&+Zr<-G%Cm&aQ1^jf≀M5-@~| zF7)TVGBdQF6l>8dK?+7fFxMpQ`-HK}E>pz$-OoyBL`RZ<8J(9QBJ_+uz=tO5^JPny zy(`O_n~R3);&{RkGfO%DslD2b@pE=4o1(9rH^5L+OGP0i`mt^sH4!4y+RKvd>wvy+@AwbT?0))g-k(P_FW;LjYJt3jKuwiIpj6h8p)KIic-_->+pqP|A>%S{3H zHrHjl^WoYj(fZNxDR}#f#+lizmz!JlR{oEHS;#s3a%9b5G(p$zZzsDDj$ilYf`#`Q zS?mo_tkcRz0YU0 zIr_c_1R(qx>q*MXy85B+YSzS0mG0*^^7ZX?vyNi_O0v^KK@T?asJ*}b{)qVy{>@+T$|~O$v)P{*LZAt1ls(MQ>lTkw%m+29r z@Kq*Rk&K6#XbM`qt^CVwyYTpfDYNgrWP+~+Z>Omuq2vAj5B%TVv|=}!M){;9(trPY z8|xf}lZ)%8cFiDDz($s#1yn8 z8piWOBtPKCEBd1mljk^!sNV_^{FJVc{Xg&}5Rc}2cxg-9w?nfPy-1^gQrGjef=h}| zEjcrzBhszN<6$SX^;yB^rEm%*H6XJ;uTY7w7pS1VBeLApjM z&JXP+?`O@Zg(@9iI=e!Z<}v@p#>K2R`Vg~2dALIb^>xCZ1at85Q;5^_=ou<%*Bk44 z>%T!oD*UCi{QadC=LqGosNB9Mc>o=J4kIKUgliUkiyk(hlsz#{eX!Cg)ZgJM&)!)` z{j1aK6x(N0-?&BFvs3#pyLQ7mDAj&fgEAv_E5DwvB0gzzr^@ATf5qM&-s!Y{fv6vz zprq#YVD$K5n?S|Sk>gX6B+S7hjA-TY`MH*e54i-j0pjOvx38TFCFku9!|$c z73NAe{r0DJg=kkkYhV8al2P2C&^;GTmpPnxzPh#whTe-Rf$8Kse zO)~>gj#`ej^MBk$OGf^M)9|}}^ET@yn5M8R8kt1nDpL#M^T1!z_6#_D;6RU_gW?i> z-LJnQC?y;_Smfr*X(dQim_jVfV7jOu-Fse5l>*5Xy-YtL(#e7uTE!3LNH5*!Md ze4hIAZK`|#dNnFmxvxGeGGXx-z5%YfP{xauSvxvmWYl_)z|CfPQ#A!>t=GM!2e6)A zg6c*EDb6u7GbjjK9{a={w!{_j*z5oPo635p)iC`uDl-Zge*4`u758KEU*~^K3L9-9$B{461}+Q76kj44#1)I_*l>|6}8V zPXHl2ym>*1(psVOQy$*97%w~BEE>@=3i^=uz7FEF(l{*jjoX|}bpUt+Fw(YdonYy| z;blG@)_xRJZPVrO`x;$I3%kcF$(jyNagh1c5hkz}NmlJMOMf3u7OM)?C3-@D!V3dF z>f_IpJ7uNq5g*PjD-U1yYKZsV;r2P;?!?_LG+`xgzy4I5_$3D!E&cFk#m}-MhD8c{ z>eA^bC$b~*x38baghj*8r!LbsA*7l)Cj81 zSN~sGCGuM!9lb25DMyCc-ATyfpY2=Ljh2`5v9_}gP+fa?_*&3c0&?fW>|a+*SQOs3 zWyJZN24;RY!=i~(h6W&9V&ca?yMNsxCTq_hAt&=!?OXo$t4vh?ki~A3zHEVo&d&t@ z1a6OqDjEB~F}eQZwNA)~95pSQ`jX|J4mS<8XJ&@Q5J73FK2A?%lJ_&`8AaGqKd^uCKscxk1P_yHdcGfVxE^60sse4BFw%Cvc7vIbJ}~s>-|sWK zgh|CN%&5i_XHt7ta)XW*ehxN^!9UJo+aYlD=JGDz^fVHoiHiX5tdCzN;>R5}tPyqh zX4hz$3U_)hG{aJ-qf+J_F&$HvfwuRx8^X?es@Jiz<73;ih|v7>f%>08grdsY&3MM2 z?aq9l6>Qxp)Ivu{Tom2i(y`}1UURn;J(f^3ivV4))0{oW+W`3cb9Qr^feY2s?=vXO zG2m(CyRIYbXLn|9x8I+gS5{?+iuJvvt3Jb?^or%=`uPou3q zLzsJH361V91PVSxCsq8gmI}*2B`jmiHy&$fBVG!-5UJETkPUXD4K7n zKOmqZ%A)TI+}(NpS--9Bp5ER&~Oyn=9tLflOWD4 z4%6!-Tj=Z8BJbLD@3EO3;Qenab3KdH37I=Xq;JjFL{ES9ypRg3k+bVQJ3zUdmS1}h zFeF)@G={Bj;CGyt(#;wFl+I#$C-6kgtPQi%-jd~7lI|bC<)^@sy~pgU%oQ`V?gFK9 z57W<9#{wt*7Yj%q5I0*j@i@M_G1^`-oS;>|8gWFTk z9Yv=O!*PEc$g$Oo+VMWC@5Cu%fA%W3*_6KDdH->%l^>}CP|*rt`pDO|Q)5b51rr=v z`I<36ybz)$UH3!VHY{6SF$MQg!+;dubt~%d;mS#%ZDhS4!`!#REW1UY$`x{~N|0?c zqN;4U&yl;Zo*Y39wbGoZtA;5i1&|HuZ;#v-<29!}Kmrvs{kxy5bJ7RJAA8*2CnptV zTcutN)F@onBg#&yKV%G#7&Q@SW^kn}Ji2Rni_@`0GozPmCnYRA&Zu+=phkHf(UtZBAI15pU$Sglbji;j**47sphmxEu?O+U3 zg?(EdwORb^i<`R96XAyEYQC^|KEsMeUy~h2Z{9vdh22@kbb=&LkP`~dn z`)pgCb@w}VN7r+h+xk|y8DM_SU-Pe>8D-^Zo6$H1R7)rPz7?aCcgI>?^kS6#8?u83 z_W7a>N1l(T^##u1n(lqs8ZJ{?vWjKre1WcW zxq&4zrtR}%?E5nn@&HB%`Z=G#+w(w3&)<@3_ja$ETXD*ds!YQyd*@Wj;k5eja-XUNLYodpyB^obKFEqtdbbHw-~W_LH~Tch)_- z+5Nk>BAJR`b`2a&Nmc~jU?2ahweWAk2(U&Q(MAs+cG~MpTVfjKZmrGo>Db{kGsD4m zqBy67@MMF`!iX;T# zP4C-#8Ar6XnOK)FRJSAOBxo*>UxU-mW-@>Mb`!tJkk-8U>G%sNVI#o4(3PKasvZ1^ zlDq}C&vETu5kdvy%W|{%Z_=lxczG0AL7e8eOgegraK*B2NWA3r-Iqj$?qM^)yVaGS zaZ0aq2^cYB%?BjCfw|~&bo<6J4heqO{IqzV4GDDlr#OCKVBU%?KT&L4zW$p*hoFJO zAxQYzv7jtHP6J<>|rR8?XgXJ62=tL!C>@Z*iLkbLdz+glG3x8__skRy|IFt-?brrcyC&{0G~PFMJIQ z+UWHO#F()J5{Uhjb_H~Rr(ssc%D?eKbc;xNJvEL$ zafU=}dV#s3t(x2z=0HO7Jf9$Hi!Z?9f5356A$uA`Usbb2^6?B{(p)?YFasxocVKyrbpVq6Y= zX;2~B0%z^RZ!y)VotuBYesJomP7`ju9GWmwk%!*5mDR~QDD(XV z0Cx1HDw^`wGmXF%yvC4i6NuU4|1y7(bd4>|91+%3qaxv~jJ@?h*?>J(mHd!KA=qe^ z0|&UN64gqccbv)-yZK`Tz1<$JuZM>ZxB~VgZm^)D)3#@ zDFcqPm?-bw35jZ%F*5h~4$5!;4*+68oxbEDf?Ky~M`BXifxRz|kB@>m08WYwXt)KO zCVHS!FZrFw```-cnR4s}n1f&~!k{2%X_)>42rMxshrUzaTBIIMwYI+8X;? z>-*?wZ5CK*c*w+rv#WU!1Owj8mo3fB%1d%NJ?+f{&F6bSNDu|l7H0ws(GP#a5l+2w znNB`LI{V<2`8kQ!EdpU-5t8Jd-r$B1*_nxpN;1+?V&U&}_l?%n^$ZQ$y)P{%JE^=R zGbJeo?sKN5b95}kWgqFO@#$$OedF-xL}%A92oimLV{p5;G*c7>IoGR4pt-N9d51nF z60(#GLpYK3bTbFlszX8rqSd;b?4?2-6o}jkLVgL7+rgd?hrWqPv3U#AvT~EgM_rAz zU47jnx*dwjGNF>USWMj=!xzqUc-*06;WC-|+@e&I3904w#(v#Gx=AtAi{!SrJX4V5 z7CixhBauMJdRFq3fNTUH;;S=`2MI3pLxVcK@(XIM1thHnR9cG$Ji!5Qpj{vvxBhU? z7zh~Y`N=Wy#?iBT+6J9t$~eJnvt$*g#3#mdH4O~+hCa7SOph zY#JEs3tIk%u-q0wttAM&oLd{qtsq7HQhHlOYt7$jB`16Bh;kH1flvps_A}AMJV=TP zuoSg48mbiUhe(?_D>pGAA;$ZF#`+#X+qo?)%P1&LwOaU&j*%MF_XW`~Hk&!$hqz8S zobAm+ATSJIDOy;L#%%DeJ@^TDcoDrv@R@i-j+n23)^s!_?MLA2@aa;89H$xiP@X`m zLuklH4#3vWz^y+TOy5U;$i$`QrzB-14s{L>whzGxqv38T`N_682bJ& z;P#N+1$?-auqc!(2ynlwoLtDJcs%a^Jo&I4hXNy&>V4W;n&CDtzj(>ARcB8hCzYj( zs}`0m*6h~W1hUd1Zplrb++a2<6veAgop|=|@4`Y}amv?z^&Z2c zF4lDY`s>evfB~nSAOG^Mto&rTrshQJUw`qYeKZs~pUV9*%sou0HA%n#X-ihM1HvGX zknR?qpAbPjo;&sSjR$__|N7*XYc8u&*FYZe$rsN&`s7h{AFj+;17{MM@A>^ zd+6Ys2QGkM@TE_0Us{=$o1IAh+<&OyXMcDD1c_ceIY)OY0&JKN$*i2C&N|qf7{hQ^p9V^*>a&51dNJR1()Bvd_h^3HHLZ({^$!&bhQLGawA41a;LlNFFydZETAxrOL%|r8uKv`< z^0mg%v;XnE7we@|1Z%IVxayM|yXne`8$PkY zCdq$y``}-H{5pIKK)@gz48Bv5$gRo~R3uu9n^bGfODJPDPCsVjq$DD6w>UgzYq{7L z*jefGz4gOuHf&w!{q^l1e#kyXJzWwKZU6P(yOylTlfUg~AG-Jd_B37y@V7vIQOfs! zdA;FR7izly_pawbVCd^dx}Dr9PWT5kp2%zazL}MxpgPx(pdPPt@>(CMA0$acptS@) zlaU}tKDm|ISAJ<*a#j-gb7IW#+8_2c99FVXK%!;)CpY9(X2m8^*6qzdzkT}U8W1E_ zU0+qPvn*plDp5q^L-zYW{ul@nBe?a+onqG{pdR_ieV?f2RJ_5dvYT+atS1^j64jRO z!%=!upY=qu1zyJz4;3$e6ln`o^{70pgbMMvLD9M(A7a8TDsH#Sh1TS__Wi5-eg9Kz zw&=yz=@54Rv&)6xc6;1{M5mDR`1Iilhbd@c9FZs{lQ}E5u&DuM(TRV4W; z8;>77xN6O&_uhQl;>zW1Esc2v#g$dd`}=z{GP0H3TAIej#gH6^rAe5=tgo%I^33)FI898R-c|lLPhd5QKD}V&NlFJfDok^tX^eN zHYsY#gmVrDw(r<}>n*odR#jzZXLodTRG+JU?4OSvIymidoO5!rzj4=HuyFL~5kjbZ z_q*RIEGj;J{MbEr-(z^liWSSh{N=B@+^&-+PyF!@fArpA(V|6nee;_Zv!%MG`sY9U znOvWkkobi=zp#4snzFJo_*ox1P7Kc6{s1_X@w`1sF$?(?fwttu}o@9ph5S5y7Z zM;|?MD82=_dVOc6LKDz}tA1j1@Tqx$i2M8IBgm7yrg9=E5jdo(#c)@HMs%!-VC;J>yKHpMyNayVRx$#L0v z$)EiCr9b_`Q%B$yJ*c-0=Brq&zw}o{$oknV<6MU)*r_olghlsraQO)`&YXIKgswo(#dk`lLh|D(*V=AdO~YxH!{1=dx$ztMbvYH zl4l(t+39t18>kxb5A|rXYF$ym!VC}sRPDQL)Zv5^Q7R-{44cX(9VqQI7db-#Z9%ZUi{OuRJn1^9XeYF zfB!!k^}AwI@s8_|`S|SdRuC{2lxGn(<&CFLJ@eo@wiwIjzH`N*s@&xpi+M9IxTi}{ zV`AbQ4tua`xmA?M* zbLTtjFZO=&!y7?3z~j&v98I7?trg1JH%<_xa8oSa*9=3FQ$=p&pie**ux=^@+loyK z37c~0jq{KH=`EMj4bQ)HeQ|#`9Y;YCUA7*X6)v3ZxbH`=LPF(+kFCG-J-3 z3nwpH=YbR!k0ZfP@GWs+jgZVpX8({^4brR*a*J?^))I-_l3-0BBy_h8-t+Aj)iujD z6>qy130RG4v^g>Esy@!?>iE<5UuvuG-F#K$&9`rf zONd#2S=p=qIu3$|Nzz(8ud)tn<#wuCD=I*qiAbbI?gmvWU^po+J(wC!9&LH?nbYbT znclNl_|@xB-&cFK^Z$PG8st>o|Iu|c=te#q_SFap>uy;= z*p%1rK5+K+^VwyYH~sKRtIfLlx+@#4d7Zn(RY|$X+AB*t10KM5DF7nk1VI!m!m0heXWkwl=25tx$Kw{@4#+vh4M`QavxjD05lD{u z>3+E$`TITZz2&yjvZ}I*B_5CG=#jS$9eCr;ul_*3A#~*);hzuR9~&FHu(ZN#Ha9ml z4D|Qy*mV^En&xk9X?*md`!;RaSya4m*>cpY_4Re{d-LU!$KL@A4D%W4fxQ&w5GYKt zx-2T%P*s*=CQij%yKv#c&wu{EVBwWlU%BUxcXxDjK&Idecig#i=gu#E`71BK^h$e2 zpp)$x24o|yz4n^N9@F?;TCJArue;8BXLyg0P3i6J{n8ijJbChDNlD2c|9E#%aq*YF z{N(@%l}~@>(~wQ+?CkvJUEer<{CIX&_W%CqM;kV5xZ}<{pM3HOS?PV{D_@0d%0U0X z|NY;uUwiGq7y`hwMPZ(F z2c4~l22bKNrwI7!-q^1Q`RKUwsTWS&di{#TgcyrenII%5A=y5DstvL!uyF85 zNO_-fOB}>O*r9vaQw!`B)X|*68PI_8Q)&IKiy9tx;mUl-roh6fef4e!GAWvE=B$Dg zSZF%i*mcleCVwP5(I4j8+DLSfrX@GTY%@ckGpl}zvV+z zT=%~bT)0;m_^xRrNYyimXesh_wVd*=N{4`{wSLwjNxubn!9z?^>o(s6s^e5 zUz(eknJ|JUx$QOG58wGTEG*nuG!;VS`JcT`3X8Wd1mOT_eW1}Hx;4^1THnj>WWIT* zkPHgw$50v-NJ;^Z8`a|6^n%6Ya`DLj2NNoE7ZeuE<8dK21v4sA#4iAR+fSeV_ut`1 z{oA`@B?oyJtJ#$bacWVL{w`AhNy*Fb#jWgCudOR zjLJ0L*A)F!c3!dzpy7i3f4=oClJKLp_WW1B`qihOdMY_NdFQSjk3Nb_KUZ9Sc}iMZ z{e`;IXU=ZhvSsz!HN8DOPd)wg&>(UUx#G$zSFBjx*V}vW;KAx@eR#^E;=;9S)-Eb7 zt*)*u;G!kw6yZ_^4i*3SP2WaedHss z@Vnpr=9y5g$qXvWrp{R_4IMxns>Fk!*l~tPB}fl`YBX#Fm$`Whe(*M{|GJ z*%UY!-@5af;i3L_5AW{oYPZ7Kv)OF48&_Jfw6URdl5w+EnA90pg=`^;q+{ZE|FUxJqnHg^-RibASy`==4fdYucma> ztq(-$mWhoi99j!v?G+WIaP+lWQb^8-C#n+iAEZWb3xi#wSw(4SIf;S$jAlw1(b_H` z$CuL81xuyW1sQoMAS7_Ss5tF#np_@+tsf=>Ba8jz6O;C~nl47qeTEJ=+AP7N&6;B^ zLw!MKkn)3+j0AF(U2Q|8ts&*y*EO13l$Md5XfpGj;C-e^oFc&KwHAI*thecUbz&_M_}JqbGj3R zguJ5UvPzWQ;MDP!fxZw;nsSU*FDZ)fF`Hp)Dt(_wmiN%xoI>B1keL9tgYCm|r!deq z1PK**%9OmMkzi6f%{-(?cgi48`T_v|eIKXwP!SDtp8&0&itM2R)w}(_&wQq` zvhv@LKmNfFedx+7uOgKn`RK>q|Ni&?=!ZYtyk!fixx?-L+E>5&`WrM;0=&qd|NQ4y zU!%#u^zcIu!AdYSK%yc(KE9%&qH4*~b7z&saW~z36Wn%mbmZpd%9T*N{^I99hZl45 z#0j}_e2k9T0||i&Y3(bnxC|B?s_gNxaq;A?nPhNiGid#RkdDZ5U#FSV#cb%|1pMT^i2serrlvym6Bk$}L&>9u^dfFXE zSu+ZX7MabKiw!kF9mOCaJj1qd4gM744ks@^;8Y4D#Wu&%y$1)iB7KizudWHXpmaB^rCX>LxQi1aNt$FgarNojIxM>*)-~IFZ5|iUd{h`-uAOF+aE*Fgv|NO&8Z~M~r)mshmo&6eQIPl%p`Yk;K;B>xFXu|xeq^EaG&z~QB}SZ`|p>6vh95-xmFw0=_w{E%|>>Gr?gy}!FtnbbV> z=&?`Vxo!QHg+D1s85y3aSegs_zUQAgHLO|S6~smCyo}D%_r4n3c zQ-~v7-xtcA+D9f5QsXj8Qq4BA8|T8wT}P*_C`ZL6|eKq-HGv1+EejaF{}^M;t+ojNmhD+(j_RG7xF=IIpKdUr^5+Nn9!W* zn1BTr`oOPH$S(+mO%X&IJ#4n&ayedo`DwSywR!8McYWs zM<2QEw%dw|3;pbT^|jX&78d>XcfWn*WfV>{Gc)t?#~&{(EBo`G|Mb_t{uP@3;d#D} zlgtx*``iBm*%YYoo_OMkAAJ9NFTQ{>G~D))k9_vCp9O(J^!gyUJ?e?6`~1-yVgyLaz?^G&sr$BmeJsc)$7>Fy5TGQb1|QNdQLb=B(C z-})w^NgjOg!O<}~bX&LxxusvItINpD{Q1v+v1jkTLxvYHzw(UHRUdZiH9z!yo>+qEvB_Q(VS?ockJAe5HnKv!r1V{2$uh%^2|$a zTAs5Y_2$oP4nh9a;5da$;U?EwOu#5~q_U#(!}{V^F3bDYkFSl1wa7I`4mSSf5Bp)E zpg0YF3x9j}tF{0B#}VyufzJ9Y7Jkc4G&$<;8#^%-aiv;MWm}JG7>?Md()t4}C_W18 z@^*fDV|sDQ+MAbWl%$cb;*#e$p(08OR8$^?Ui?I3^Ous>gp@I>p?Iih@|pnZH3H;% z>g8nUUjl%fp#%WAUb+@(A5m2{te>%H4=CV-z(Z;TCFv27!-~LjZUHNIB5m{{alRDl z!-D@6g}FsVoH%-@wYhQQmR%W{*(oV$7wc;~I$EpGoM>ro2xs=y?b7I>7_#l z4~klcxUtc(Pk!PPW7xMNF)?ZLmMs|>8IaOAe;!4j@9ysU<~P5Uo0|*igx;PW_|^N~ z{{dM4r$60$?>+az!V^zCIc~S#a?35B`hTDP%U}NDc6%G4L?2}Xy!65gU;FQ`U48Af zzx?GdA=Lpe%>WR_XZ++|8 zXa6IATUv^SSv|cy4?gq|WOs&!hvMVow{6=FKPf3G4?g%0vRlc*61eTq#u@AAq>(ok zEnKALcpB>KPnu90I5wAzm75kM;B*m(-mLQ#6U-OPR`YFlZdA(KL4L|wq8=XDFU>4yuE#>Hf4K{e5i_kR7)Z|t@MtknT zcXvN~rffytr@nTHHO8{*x+SkjNdhjrarw14Ir%{E=-7y(pd@|kRh3W?KKh4!E~f{~ z7NSOOhv4Bho>2 z)`GOnH!OSQ?}tbF==yHK6Liod3U6cDjJCe$vyN#^BVbb=eP|*_KU2rDO$*}jX!D&n z>qH^^!JK#_p$ZYdCPzB&Pk$|x~2k4*Gd{3q23qFZ-jTucJ@^J+j|@8 z>v~*H_sVsJS6#ofq%<=vJ>j}rmOt?j^@+ds#$`8u80CZP?;9N*b`+JQUvgzdY@7wM zEM9$|FY!|@am}U$i8%4t{x>fOp=v&I%6T|u3**S=*ndu!Z$Y+{C6|`NkNPXo--u>z z%-oGyUks%6L(HK9)K`kMt}cw}JwRs$@z8 zB(j%Cs1*6{xrW}J&gY+b49o<)Z=Q{J7-kNf)X~Xh)mG(S0z_pi3Y@NgVS*%qa&}r_Ua8c+;GJeS3*{U=lS=(_eRJ-Kl9Af*BIrcm+_wWy!R_#{cl)+ zJkOy+hvfQ8F1_T^OD|XN3GeRze*G(T4P>?(FE-j@Y>+p~FDSVDiYvN%dJY|Yn|zy| zmPXFs)Yx?6d*6HELS1H7)=z%&Q%He)?dxAZuz!EU#Rk}#HY-Yn=W;pKXO^FHOssxw z#Zyl{0r`}hZ@!rjDmP(om8YJ15-=DcP%#Rj;(2S0a~osALYl7PL$I($)EdP5H}DUOjQB3Gm#_pW3kN`epC=$ht$X zotGmpgau}cB`qWS=sWvbn^B^Rm20+?mM^V8f6nbzUas=R%iy;Y6JtRg6{VO`cM;A} z*IXUJ?SH2MeqbRdJL#UE-IS3QfAe+AWkTgsAKI9j5?5W<_3fX&IN@+-r6=C|i<=fL z$hhIECHFma5X?3hdom9(Hb^z9bHeq;e@=Pa!nq^O4b@#0s|ui*GCnb8Y-9p%um9*e zI7WK&+0)Pb?GPMzuH3xvQ{T80D(d408cwOw*Ut{3Np3mSb_P+rBqtLGGCI5eqI2A( z{_5>#&%`IkZh7xAG9?-48fmKTT)DFhmWO)-O#m=DK*j9R53T8F>_7DUxwu3+QXL(f zQlCB-wHBjQ0>;!w!<(+4qs@0-4?TdAAMAVXOhQWR&i5=O^55S*(oo&Gb_bIGA+`Jm z+vp*RhsXpgjM($)dAWRWZ$o#-$oGD613<(6Y*Goyk6S;o7WREFKYQk}2M)u&Z~c~% zFMREiP1_b7eyicwp#X!mD#Eeza%8)Q;=1qk^I*!D7h|^RnK5_O^}h1ky*od%$rgtu zwyp`+saI>(+nk270I0K`0CnyYwcm-72ZiHQz9~TCo1&=!Kyxer z+O6nSquI_t$S(qs#{MGlyJ)Ty56(O9mOmbxH3fc9B6>lT*pxZso-zj@2LEAT2o*Yw zquFv^WhqaTcRi5n^&Tp6y;KH3#X@`OP|9Y5FiH0AH+O&KOJCl)Wy?oD`jH18{Kwz` zN?&>TWtV^Qlb=-YHaQcy-ripMpoU z8j}i;X@J`;Tef6pXTSaSTmCc5!<+iS|Nbv5{PVHL?z#JJ^;hL(i*L~4(n~JU9X>uj ze*N{=UUB(lS6_YgGtWG;bLY~Iv-*+E^kx;op;W=le1lDtD&W==|4)`i;BK^+xc;8dR}og$pqGUw6R zHT>2)NPw4B<|f9W7X;ELB{+WFt9xomjIZv#k;BK(gnZqqA}|Lq2iILOOgH2tDBbNt za)RZyCRw*5Tf>sIg(S+&{uj;>Avt-lVXzks=a#OW1ES01cFI^SrK3Ocnm!AK=oVgm z;P6jA{NVj}J^P0*Jn@^`|K+qJ#sxBP_JG$nj15xLv&CwD?(Y4tTTjlQsZ&M=LQKm{ zFK_l+U-aj#!Z4kHqs>J)71{X{?ft=zHUk0FiqAc8_}jNV^!vM>`}rMD{P_0A9d^`x zL-{vY-xK&Se@aWTm{n?8P=CIMWJXELqLDCHtf771>o3v1@7=dA_G5$lsx`s(ePUK_ z0?BEA<|;uR5&!@Y07*naR787E*YMQaalVY%yhT17Z$Qo7`ujie=p$c!=AU1G_P&oj z+FeI&O{0-QsD$tYM9C43F3jfSlW>F#QP;~$!KHejsn?c)R0i5(fv>WtDFa%bv5;Q` za_kKeNb%2r67owl*o`3!&^<=BMt~tO80Ih0`jn{ZC7%)?sLHZWiBgu>6oohDfMHn* zglJd}s~k^EOdLOU9De@%r~6)c?Grc zU;s=RM~)n6ZfaV+X3c_<1vlM%3#@tS34eYYTefWd*)M)!GMWDKpJ#sZlOOB8J#q4+ zI`KQiz@X2IY;9@&&Hw%Snrp6&iLqUK-F3v%MUKH$U0p5j2`QzC)3SbacNa&SzW# zTDTgQXtldTXOcY+mU6*Y_`ffl$S5-XlouF9S-WK z2*+%@q0GqGO1(}>s=HA`=gke+`3(!o{r(n+@$Qn6pO`7vd|YT z$^bZf93-{@(#cf0K~>z_iIkx&#zHFN5^TB?UU}d52d zL$nq*wcCwJurA%a02aEN`&+7wlL9(Moh|3O0FKGI1n=Bdr{6O%J$^M_f9-Y&Z#{WZ z{Z)HCwN*8oYVWA;gN33>I)ZI)=mnv{tM$Qbw!YDHN@S)246ojnEDxS~Z@d0-I@&z+ z+LY4Kui3FEEeFjmYmc`zpYQSB!9MD2tVZ$=8Nub7ieaI-uE*aQeK$SlrLLt+1HnEH zvj2RMuo)P$RW8d-!r7j>+UTT49=4@rBy@M6%9vP7d_3v<^ke=TZe79QzHiT~wL=32 zecz7D%Za$|)p%2d)Qrazd!mBEPVDepOHOaAY(K# zG8_|=DQ7sWt))|fMT-{I*3$8oEyh-mUjPuxa*ov0*CPY;>Q$?Aa&qLeU32v{aNF3} zC>Vuhefp^FzZoxLz5JHtLjz+e>GA7#puk)% zhpSB$?~5~W8+Rd-%E-|8N#CJh7FFe5_aPLK|MWY}Izpwrsc*%`Vw=r+%cnQJ`oxLk zwD{|A?0$$2Y29?V(AV3!u(TpEF?D1F-9lO#qKNF{bSjmcl$xDeu=kDUf;uYFFxw2s zY!7^JXj~wDLbUP3$69~&gB$j~U4P+XZ*Tu-Vq)y3HN`8IB8RGRyR)Mk*`V9nhoJG= zY%yK6tD>o;|6Fb7C0j}>%aK*FsYOq{FI}8_+bu}!4sB|v@3_^bOui6bF}2v(j0V3^~;qAH?nnjCFB4Sc;<1}%H&&D!ffybg-fbst&# z#?z;&))XwL$b$8)4SmFCMfOU(?!#-I`G;q8Xace())-`IYHQMq?v?LfMI`g=yUjX6 zW!092nK{UluV@h^<2PTin8X}A_V&fzjv){vL{X7jYNDMVP`|A}*ZRS0)a5%%EjHAe zM~&7%zb}y>}^Z zGQk2#vSPJaF1>jL`D)L9PRkB?5NmdoRjfkuEGUpVLZz@QD|bQ4$paT1c2{zG{4Jl` zoQUOr_fwNiLghQtzLKJ>uch}FObI@ICW@ic`Jq0*w2b&$Zd-fg?S_u_5wn?JzAAs~ zrDf!+8oX!>qDAW9Zu!ufe?9o_@R0pIx309|zOS`OZ+3sr`&N>^@5H+;-h>Ji*KJr_ z;Yp4eh^wEwFlt$<^^{%!%n#`LzL0O+5@TBR-eqTBs~zjJTVu_uZ(Lr2-EB|5QtKS2 zOL%k2GYXcWg-z*&sibDXssbDKIPa|MX+7N`pDiIXe)9)Wq8-@x9r4l(tKYkXD54ll z*O_hRwKpv%Umg3;X~7-*Ef57N0Q3=6e1`lnKFWv9P^L?OKbxV-f5o?5QF9(zrOMYL z)dRB19Ej@t(D=AnAym{)h#?9Pu6B&vJ~c`uS;<^NbD4+475xjeGa(C6!lQef6qUt6<^4{{7yMBY+x&n%dgYvC)M1 zg#Y>WcRv09K4rJtlT(tv{`I@ySFLTW{sVJCYiveFhUPa!v9GVsd(UV8^PfAuaK|;* zUW>}lK5LZd9%1RyrGL2l?wHt^hK7b;{OsrPaq;q34yQw%C&m!x&Yeq7O;1Tpx$B$X z`qnq^8XreX+CTA$Ph@5yv%=oJ`^avuy!`U*w|{ow!iC@d&UbJB%%|aQP)@%4-S63K zsQ*5Eh-R~Y>ZvC`eA{iw$;n^*;+MYn{qH#)4tV`*)}p@r?+^UV@B&VqI(6Yf-4Yxp z?d+K|H8oRO%oY8Zk}8T$J=SoptD(AcVP#I`s>1KwQz+NJ@!XlQA!X*Ue0>q5KmbY* zdO>i9l5%nH^XIl+RRO20n=h}}e0hajUwgW(?o0;=D>^$`oKDA*o;nT!MtXYu$KJnAT~kt&MuLgeH}-}| zsE9yR%oNHKk+0Bue}-v=x7-4i+`{RnEV+~f475cI-nzrDpWk`i5~y-7dCxNVk?Ws( z_+3&sf2zIibbHydJgEP_^0OP{9iSa^|BL59z}Rp}*%EDtuuE>BKiA&U9|EBQsBu>M zA4q@FZzEQ2MFS%&ty0(%6$$KzO$*?b*W#rTE~oqCZUbkyr1ZEO@7Mu9_EATy`-v2U$4J43UOR-Z>G{Pnhg zYAuBZ)yr+r&9elJHW9aRWOu#CXta6dhgMr{iW>p6gJ1hm+I7D5(m79H1Jsm^*bjVR zJN%4~I%J2unq#fify=*EKa1)ChnLoxLT?)-%!(vMsy9dVaHQ@|V&p=%r~`|c-@3CD zesuMX^*t{>dxn(Ho$9DL-BGbDx4bg@J3m#jt3p!bjhCuHFj%#|C?yqB(Yr6WgPR!3mClFwCMw@;b&sZVT-ep>DzG6$cg99$ou3k$=UJ| z&8U9i`r?H+D)q7doM}DPp}B0|*zmzsYDHv?vB;-A_0l<~54)2e==(k}i4M`}AN^Lf zi@~?jR(>h{>;pLUc{DyKsy#W?m5@$>ZsGs4OQ%# zn?zj`l7|Z6fi>l+;K?VRyzcrNa&vS4_{Y0@dU`rKI`Z=J2$^yI{P|a2c{$|vg*?F* z@3`acd;YXy#ftq0-m0&!FE1}Qo6SSRLtnb{PXEVl+qQkT#O@GGOgPr9UG2SRZ%@zu zefxIdWG7EQ=`-Hf|NZ8-klKL-$acJSP?J6Q$3OnjKOcJx_yy!??!M=b-~8scF1_^9 zz55Payx5SFos*h|mOLCke*D0GN~m}|o?rd)7r+0*A6Kqi39tXmnKNajWn{PC|L*sY zMk2d)b#;CJyWjoI?|ygP_1ACNvbDFbx2y~;t9kyp=bnH5d0*G?)RRwr{Tttag(sPN zihqRuN<_IT`~7!cy6ui_t2UQFn+QOmr9H1cdGf`79#iilQy_1>8X{>oI%NOj_g;Gc z?OPT~AwdK|IQ&N4GY`HC!VSUWKDhtY^_zBGdgV<3&5`<#zk7fTtmEPn@(M~0z4dxf zN6l5t0UwgN^my>1e)XS}Q9x-2Lm>fy%k4RGqGjpgoS0Z8cvoll@V}lp@#fFBeOzmNWLS|k7s*}UWyOhIl5hWhRQ z_utQb;v1L9Q5;8y?f<&>?V985@_YNhFW$W2 zm)sKHohTDp!`4UdtUxjn`oLtaN_pj(ISbNZq2^HIxKU0NhusDHdf4d`HpT7s9DMf7 zYr%KNbw^wN`LoyG^M!2*DafkmoNyg@=JcLNkAhGk3LH)oJ9k=N^!d1>#MT`O40~JD zMuIp+Bk#t`=xFoMtJAndOqTyRN&fdfd*;Q5j(|{s*OjY$3*>tGpr$Uj(IjoMjo4%` zW)pGW7wQ9>aJZWr`wEIuSrE5p(fk@_^aLT|Jb^%yGrH0ul8)`gZCdC z9ty7S+j-eyQrPoaEtoQJr!P6_>7Pur^?ksw?-M8M`~06$5FB==E!G-?lYqj9*m$(% z)!*%(7>R&z=tPGY_I+To*7vd>(r$}H!BP8^sQpOf1S!C%-rzj?dKz)ZpkDG#A!SCW zFhncZEPARCedRCl@+krSeNBMIzXWg!SM2S50(_P^`8;wtj7i%qx7_soAN=o* z?K?6t08ZNR<{NMP@BjY5ZlBU*`<-{*`O+8f{MaWxzH<4>s;a7y(UB9!kN^2kf0Dyt zg#}MO^&})zAhFSKvBB3qaq)3rYJj9mTWjmrzV?j;B?~GmD`Cw*{{SSMe({T+s~ueS z@8AE?kKFe2pZ}t;sA&EA^{}S1v-5$!{q@mDA61|2^*3Js?eBhj`=@V*bPHizcE9=N zk8xBnUwH1hXTS92FT;t!%P+nN7{Z2mq#@l7_uqfD`(f+8yaj1+DBsyU=yvK?*4+EQ zuLN?c*Sz{M;s5{;07*naRGv5iKfbj0IWoINMC);{@#(Mfk4Nv&^vnSxk@TZi0N$~>}?tJwY0yrru(}$_&ZhY zJI%lN=p(7QN!A#1SJMEPn$VKg2ly2LMy)T-4yOWuAOC+bGCIfbL#4IEFlnv(C(|c) zz3)K~81DQ2%f7xkf1>@nAA6)=Q3fPby4#1keanA<4?%XaP(29_gsZoer7;R1 z(napl6C-^en1U1y4cNc^rDx(2tmzpE@d?)czH!(O3LgF0AaQd4@Ao})-`iQasPF6U z9D%)|;R!!=>p%Sb+^>K5DhMBxzE4zZKT#R8g-wd)4AeRVk!e$EedoCQzK=YTU7C@Q z5kJ~D-q+agwCkstN1r|eKY^UBv$p4V@BN#vZ|{Bo!yv3s`aaQEhxi%!P_Yo@GN(F# z13vsnRA)7)H#m=`UV3>A>w%_h_PX@>*%_%`t<2*uDQA(zo2Ac$;F0?vSW%4f4%?yfB)<8^769S*jVq~M#n~VH#Imo zxMuY#bhy4K=|YHQShN zPn0=4@+A)OJo<-M8{Y)Ej92^#Q&{Q``I64AVfdM3H@DE%IoQ?-rWrvr2E1!zlcZK@ zK$Z@xr>+1{!U-_UEXY$QF=8cy2^ztXHO@ey!KNa&GRDVT4LI?*FE||Twu^oeSxx2r z4@I;DEV!~YLDUN_C;tH=6r#L#lLu1k%Xcc9r^C|v2tL-fPK-JmPKM2kZlGfrgaZOz zovs*^1jXW}SeL{p1chM;6|&aCOUq?h0iXbWSc%%f+OT0~lrjqG~8eqMcxy0uVf?)m%=u#cEE(iBBU9hB9m* ze@QqDJ-kz7Px0V;m!AV3^NLX|wj$+*X|yY%wQ%c;;c0zN^-FUPNonZr)LMZeSu3|RVh_-0tr?LD`TdAU5hOMbI5ZI??T4iZ{o!=F ztx$>*qgjelamFC0w(mnDSfmF5m3mOvl%#KtI<)%C!8f6;IsVBn7;dY5@HC5XO13=9mB4IGe=Y(<}CU|0sa zdY~!O6j7wCuIH3`&HmagCzk;O1Hq{G6J3@~fpg877kg$q!8PI#vr;PIMS8a_%?$Q4obtN_C- zLUv8yL^;}t;-Da6x#c*1WB|!+{-Rlw+wj6b+6xyUxrKMiaV8I0r>Q&&ASfPZAd;8o zY9MN2x`?7z>kGK`M{(G52PmlsU(y@vp zMMHSZmS30*q8G(@L^>o%k$|-yb#@YfFjur+SfL)Td-Z)}DG+=b ztxp$!$t|wx`{XC)3&cW*i)tHSKxLq-7Y+C4H1(oky~f`~U7jML;s=anVPJ@C=zK?v z%fPSJ2T@xNT2V>L&-l@ys0TGYjY+B(l zx`)|-KbsI$>-UuKBOdts+@ZCcg%b*>FCVj~hJWSK0+o2yAWDk_T$lFtspi$*7ig7gf@WOeWeq&;>yd#3;!MnBNnDU^V(1P<9fsRGiE5L(9zE zgGV9HAg2OsEzZcSR7-}Q`t1~0W*RI5YqS|?X4AZ|ZUB3N6dbGk5Ncj9O~+5>En@|* zz~8ojQ{V$udbHzigczO`x$^m8SA%7|YLU$>!!`6PFI63V3PH>(H685q;@l@Y7JX84 zc8Bi8gwQ9T8WtOa(twk$+u#RyfL<`0gQC>ks)&m1^jW(%GZM!c^<0FJv`4pk{JbsY zhvw^zyo+R90yOJwMoH?jQxqw%*+sZsb#Uj1j3J)qqnlLx4$esNDPQ6Uzf+DmF*b^P zAA&1~P(AA}Ykxgx_8|jM^@b8bQek`cbx&Xk;5ptWCDsqzAfK_4h~JNd%d}S65&@)W>Si77Z-p5N)G>{_3Mdo; zG%tqBe0~5W=HIq zdLkJCASaK{If6CwOw%V`sVQO3cWaMc^|SCrXsQM2!$^WtOC3rKw6oL2Z8vW|*}kM^g>>G)y!gEN%3(4^R zQ_5-nfaaMEhD!`9L7je{Oaj?t5kGBw!y*~gHW*pUN-&CXMX9t77OvBP#%4Yf)WzTcB*(> z!X49(y!&b$QOgL7)W)0@AhGFva$3BppaG`{eBe}rkswcko-e$$i!JkoY5be4J(rQZ z+AjH*WZWWSmG09fTq2gbyU&}}oWn~QqiKUgN$8bmWYtyx6!dP3K9nWzY|zD8(w<2` z$$@|-;`8qp=B7wLx2oj1wk(xhq@TjfcZB%dmdKKp<8k5(Btm!K@;jtC^8k(6uM(}w z=|*uX+3_laT#4_6Rj-)v>NI*Qbo?s&hfI;@K~9O##spGL2-Z&2-swcs|ULdV*3N8YbRPEn+4nAGQz@k42bo? zYWhQ5dkDMqROFlaW$RDS%9~^GSXC0>^MkmXcWJ7Z%hEEwS!9XzAE#K^Uw$v(v9N$h zz_vDa5^(x7Mz@`(`xmhSeynHYk!#WZquVUcod6|XlcqThWMQu<*shPns}N2{Mg7a0 z93gV>;;bJLnT(61$Vm6SX<E=`TOrHzJE}pcUnj#<*&{^A|&4JE1wkF zJ#xKw1a&e^A=QfsAP!FIx(T*W$iJvB!{6SB&-1>0JIdco9SogCaDvJ<^jiFkO=w=O z5zXOe0SPflz`tU=eT<;Ec+kq02~$*-`QnXA7(<1YgfF`4;)Q5XrfA-~V7?a@;?~a# z$vGlIns4;cQTQPAM&f>RAqtcDvjXnTjt-;(AR3D*8jbdt%@xTJ0LV#-L`9*L&2ONI z_UjQp%3sl9Y~eD>|K;n58OdF})LB`IJamjGwfeMJ&G|HT&YVcaBDS~BOC1cm`v|zK>fRr(5Y~LX7I2U|b0Rz1Oe}!8(mDH}>wvZY zR*Wke6_gP`(y?veE7=IaE?oA#v1s8k$^YeG)L@T5&q%qtAZ@o*9^B$gfF!k`+9BC@?|C(P5|IvYJyyUC@J601o|)Ycj!;T&Q#K&)g5Rk6_Kwr;rB!V0VVAsP1s z=h{yIh>YVW&uBmF%f-G2ZMFQic0uWBO12*#P4I@Z&XuFW?Cdv5H4zhvyGmg;6lv>} zep2auqAWj?hxRN*{dJSf#3aY z#!2KP5kU-3fr<+F$8MV5$^4rV!Q#=*rPLE2-u$d7=XK}4VDFw{)vH@j`mrPJv_6@c z*Q-V$a+UBTolZn+cwxn=kECv+w-R;|Jsn#;+Gm|6dL0MNlPsA!pGB}(|7X@t1|tiL zm|kK?0Haij>&!ski!(?RT$z9xNO5|&*Ps6`VM@HRDxTMd9`F!XqH3f%L z0f`S1jW;PWs^%yA|HlGam6va`)UU}}RY&;w>j*j0cZCG6Za^`_TpzM-tNsP+T1kER zd06`@9?XU4fp7caRUY9Wy^mtWuA4NR3{psR*_hp;O_S5P4%UxY2Rc2EV@rz0@DtS& zUS#ZE30%EaQadE88ErfI^%Ly08((h{K`Vj#De^C=&h4vvCOnm5^zF_G>DLa8iwLGi zeci_lz(aZ!RDjuXxyT489=~h{+*z_33B$Nl)na~m z8u>U?@7(PR;cwhsIa5jfi-I9qH}`2CyT5yhubc^q3R~!bQWe9{lkg>xP4wO{S1K{$ z9ewNZPNN9ZbYb2~vWhEBM+b*3$kO7Xd`tMrz4AUane^93!xk>Mt_N@vLH(rV?ETNS zNv_T6%f!4{K=abET-IKaJ)xHsvgU7@<78mm`l~>(L`|z<*jI$!_2EcgwuRd!W>ff2 zviP6etM5}&Nq<2MTLj^{(zZs@>)hvGwVaz@JSK~+D2^x|_wc-8xJ80D%wzoA0MG`_ z4}TsYeRA}UTwk2t+Ini}z1l0Gewre{FaxVGH|e^@G?E@UD^;i+BkO zfK(uet&Ke=_)$ueOF;%_Rn{Y(jczFT`wT-MQPqw1EmRv{`{HBGzV}jaTo|VPd0N}> z8{S=;++6nJ+T)Z%?iMJvm7b#Sfa$|I?fmUIvJzxBsS~f6Sv4UY4N+(W)?0m zG7mi*^PIAAD~H+I9a4XlTKh94Iy)=&HkiK}gq5DYx4p|`m~Okb$J8smasBU3*zZ+% zUi;7y#Akh*+NZMlBT_B9i`5B%@@L-!1mRxyuVVFgAOJC9U@%@`A+U*q$d>Q?ECh>` zt(S!;-Kf4Fm5QJJ&qI{!$sm^;WNAgpv$pkH8}vuLwVS_y#kLSHuTVWbJ10+BDW5ne z-jK8HWfzL{inzdfsMr2)NDSG5?~hN(lR?OsVL?9ylq0PQVzpb+Oj5j~P6cIW7!dMx zf|3u2x`^Nd?4|(Yug7r2I6bhnC8=56l|OS*^DBII36bytuCzqr9MiL z;9qS%>uZQ{R!`|+39(V{9SPc!ue!q(68ccp7JNfkXJ=oznF@9uh#phO9mg{MA~J-h zUQSb@5wVWIqAI{6>N}*vKZcl_iS%G}_q*|kY*6yO<8=ex=SJN4gi!jX`b&JG6brt~ z6ut{RRgOA-X4v)m#7+E_I2Xvpo$%BAF<}ccM~oPScWUgRLz`V;reJMW4PAPkzIVPD zbEezct0v{ktfU+M9HdlW{p_s|$k3whc-+;6r%QKsK}z3nX6296Jf%I<%tcoBv#II_ z1q0mlV)$;*+CR^=!NT4?Beyly&5u<3kEnRh>&kY?#Fzrv%d-!iA7&NmQn0=^m#!z_ z(bLcoB3GUEKA`U78)aXAb#>AjfJskE&FaiZ&P&bDiT$1Yn{HT$pU7;t9fMym1Hx`S2{;!0vK`HdywJl{=!rO}wJq)`GuS1oIb$AtKP;&T zZZc7-62HsmT;$s*mIb3T>XVbR?cx<76J%JaOvHTh%XQ0tRgBCc(kk@9Z|UfUfGpD< zRgw*Bk1aZG@F{o(q#>v0Ia+llkyYR^*(@(YBEcUqV<6(io3rlUZ$t3wJd8%oo&1We z>+*a4<6eD*i6-s1YLV!4SO|;H+lt_MV&VLZBI;|lW9HY&#p+|^MSL^b2ndWRW0C{G z{wPt7QgoNiC1Hi8fe`&fCk`C2AmM5Gv=HyeDXVY37PDviIA@;_vllcJvO~H6bl)Z- zxUZU|hH1N$Npv`P==KqD)cH^<$0F8mRb7N+s6Fzv`q_%$hwfdY3>O3u0tju@XJgqY zp#b3qd5Ym)ZA|>YpWK+bhuD?Zg8HhJq0tODEh+Z&`t4ez_LFv8rc($Rm~3@@$E+Xl zo&#)qt`v>sPa}2CQ^k$A?eCv=w<;9K1_)aEgM14Ks2#bwx0stKVIF`(otFw*+Mo15PZ(&xOPbPMd#(c{+gL(!Y&l-wOWSQZz9saFc;019SD4lb!Q! zpo3mP%1M=!wHj(`!YCHywBJXdbO)FC6+IiB35YC>;y!cq2l_QM_jPrv&(MXP{j(mA z0o~&*1@0{r*%TzFC`BuA5rmdxRS_W;D$<6zc-|VXy?+=qp{IXeGN`Bs?bucVW{B<^ zZ9OyBm*{InLntaorhRY@D)O|?6GUf@>o6rzLeRl@X<$x} z*cWc7hf>jpubr#!9bf&q@90!E<`}762LlnLr@UEGz?Trr9wQ$x6y@>fgq?ov*tP$aeTUE)(yk|G5 zU7!}G0$HMq4=#qrzQ+4s+l!gA?3=GGffd)RZilx`xQ)4oRpQe^Qsxz~Q^Hh*1sCEY zY*yRyg($RkMWfl43P9L(COZ3>LT=>7em!ITnxVoovXzM_ZfVsNxV^MdeU>&_steVa z){qYl#!k1gA!Vvi%wEPtT+uY(3d+JhH8@+D_gK^c@Cx;@Z1f`GJ^NpXVa$_dJ2IcO zko0E3NfXtRyg6`{u!_~J_toAGDoW!%nR*OpEvH7E9N#@v6kP%!U`3|5h2O{HFRQij zKZ_nbItCINUib3mVA@Md*pxV7r7^2SpQ$2$h$gnd9kp5`GfmR#@uv`}LdX{nKTbNJS&A*Dr3FoDc(M*G1?i+UTJoHb6RKwp;kJo}4_!YxC_=;C*XU-&WieltUBEJUuMlAGPe7 zQ88{9hS$Wh{NP<0|1?i5gmw$fR1Ny)(Z@OW%MUA2LVwnjMZhYqMzJ0ZlEr_)qce>0 zRYokgZvfT)ULZsiQ}B>~=`5fkD=F~fYT%4Zf8a>?(!YsSAApdq@oR5`j**Tfub&fh zey1eOAtkqy@@mKoGK3hIDuHLLMWKnIgcwB;tD=kUX@?#6t>5pCJWlIi+{e;J4d8cb zb;GK@bQZ1>l&}VV*R|13VOMmtIi*nBMViKej>DvIaRR_}zIO|7I{0s;zV;|@pjEyB z#>1E7d#WiLs$jn5NQFq#hw<@d4P{-q=(9G$=k*=GH zr#OKY=@K83nhLAWSd`*;1vzV~S_YOZTw?Zm2s-4~3;R(|xH&7DOI}B_RHfgA`>$o< z@2HHDRqP_v*}Wav+%Bz_A!=@4Ni_IbY@p`F5ELQE~fmY@Cf{4qS=1(Ooj=T`g&-PrwtW8&9{Hq;Y(La|VyKdlz^ zj=77gUG^1?q>V5<){gvD;tZr&7-7p@kBLI)+Hu`-YiZ=+;}1fQABzb| z%&PY~2eN%vjH8)x(Pkg1y}8*5eH^5L?$Ck^;;F-5EoG4>NAKB-2b>~5Fpeh(_Y2y1 z6c`6yn%zmWZZDR(3I`+Ze8N?Hx3^0DBu^skXn^I4Qn*Qu+l*zN7(~O{xX!Z1)vOir z+Wtm#J_s(s8HOWVH*Oj^u6D;p{?_3ANB<6XPv=Qn+_G!Tz8w3pXZmUJ0wzgY`^T|! z6+dr1<2fqTY%k(U`7q0_HVy*GPmxVxE(F7E4kM&jsf2?2V1U3^HQr_=g zCh$_UA^`xU$Wi1$U6rgiZzt-3(==|u;_^?P!*}|2IpKP8x_D=r1fTjoOY#c3r{^nD zWAgnU3&7J{C|8l|%k~1rIxh-RWdz7v)?V}-5H_=V1lQAxXR&%Do7BhTS13vQ>3wt3 zbDf%5`TOeAoN05ca9I9gf^o`&F*nIBp?M}ferAMqFbSBI%Tg0w6AEfDM9xWMYat;@ zor+2~BH(Jmb+};JHx-B_FynqeYS|I`$uX$s@?l9Bq5x_kJ%Wf~z^m8Ici@eC?K9m+ z{RdVZ?6J$qH7$dkMErW_HXtpdRD8!S`oHs{4^zlZQZ*z=p&WA2n8OmsZqYxW;L%Es zWIomwaDiK+)TTWbF%K-v8;p(kePst#OyU(k0m1=e@4qd;y&y0xUD(9?g~7;*8*Gya z=5JbIzjp}xZmTd_c6sTg!D>w93Ak^*Ik~D$f=9+vj0ntJrV6)(uCx+!uP5bQl4cBD zM89BmIF*Nxm}@%v3~@2#=XR3%gX?3LtQ9% zT}N4dchGex0u2IDsg_kDpOtmm2b>0tqH~v>$X{)?_j|5IP#+y3J!oYRi+V!~)S^-M z6wkGqrIXh+IM;LSP~S;cYlRWJL+NPmBlkZBM?zF|VDWuUYf5JiQlZWBYkgNxW5fPy zXsKERJqtWtbQ-4B`XIHP?XV7KXqYU-SCTr(f=6bVXcPzC3d+9BoG|*)ZVxOD#du<} zb9qJ#2Qujn;09)R29_Zy3;ADT-(DR|gMwzbkHfudSt1S5&p*dsaBnSg4YL-U4I}s_ z01_DQcyU#Of^Xs&Yg`gv>vk(WnFn_;au;ARkfy|Hg*E8u*`D`ekIT$fe{f167Nx#p zSXR;SW6A~cZUl%_wCTddq7wba8KXsOUI|RoXvi(3V%lI;9BwRI!;1|66&YMY-j0)i z+Keh$`CfH^QQIW&k2N=MG#Ivi5jF*q|9HKn!+!L3>^d-JCnm}0cg1GmPWpM(`fVk} zTB^x(&|i9zr1#sUJL$MzA$8-@LK0!wmc8mTAc>Z^mUUl^fzKbh?TJt};au(t-+n+w zG={8f+{LJ4_k)u4B6^QC&cR36Wr53l9u>xLCROww$Lf7|`jsmGfHl`D!;_#TBKKqM znT9IG3h>(n z;q=F~^JKQHSg*c+BMUuZssOana;pd__m?#q!`&1^?YGz(nV8EO$y=BaG>D~!=pJGe zD0Tptpj`{I#0#dc!l-mU7`qlcVohY*!9I#CVT_Td=8{JXIjjru_42 z5DwBZ-l$;h zHffXz#dJ+c45LZ7-?5g5xNwVJgupY76My0|X!+BL@EU)d#4yqFeUdFjPact+*gg%1 zfACshRKOFSiDY0AGCq1yUnJ_N(&~N9x)S#*G$|A)2ZzJ}pI{qa?d! z;ulc8;aKBy+wmRbf^fZ8`oZ5p)3CfrZ2g8FbFolnEUFwwtGOT3m(h-uC}-33C}thr z;f9H*hIdai8mAbfR!**eI{gAe#&F~W(bLPPn&_D`tp4_-W25LIi#>*f`7cBNgTFQ?9V$Ol13bZ7< zc-BiXYB#8Bc&Rq8p}50Nn%7VZ5KnNZM^)n%hOeQ7&8Q9QJTDca1dxs$xcTQc>E*vu zxtW(r*tM(MX>~6d$Z8bp=BrA8&?q>WVU;cw_fE{h?2iTvjR*I<-YRO~vc(#=PkChKmSuWoUUe2tXNu%qWqJFE>04 z{+6mwd-d*2CyW>4Mb}lolDp%Nsu@^b0|1G?@pae;mL^c9LZNRhr#)Y(PVz4sFRe{a zRHTX$w&`NzxFqYD&oGx|p4Do3#=S*oT3nk2*}Qw}+|ovbuDe;VE_Nrpm?w?i+i+C# zo;4Fd`t_qi4V+BWB%a^!kg*SSR&?dE&V)7I_K+XpPXA0kDlr-!Uc#k|D)Q^O#FhB64B6uzV_J}@lB42DSp8CdO{8ODoT8V4rQk;flyjnGLQCk@|YJvPz{m?20lCY zmp2p;CoBr4PNrB4l5HtIJj)XS>1a{_fF6cXBA+0)hS&RwLH6OzlY;IcXmVmN(pRh^ z|2-)6Q!<~t9%itwfF&v0$MP0lBAb0y{^Uk=#@k?(r+XLo7V2$Z5k{XFPl9TXKEah7 zsHs*4`E&a?t1(k}de`zrP-e)&l;{8*g zc>2rCR~f|cE}ymPNMGZICzZZMVRd9>GMv}uo+W=7Qm)dsz^pP(5F%1#Jsw2poEytG z+Rol*bQe>nG7|!Ay{7m^?c!=9$ux(>u6~$uB*qF1uBW|dceuJIntF(w0W*EcqBJmL zn6^&XxK7-%@`glO6k_|L zy@`g=N*ggW9o^H_r(uor(s==h?dq-9-bkk^fqYR-FT~XEvB3k6-AD$qXv%|Nj|y$7 z%H^kwf4gwu$>SL58N`^N$0S;ZSo%DPvH`BO;mbO=8Eg?(EUlP}PM{$Df~e+^Hw`cL zm#KeA@n{a`bn6yR3+D(XLyDFj)AQ z(Oj&K0o10BvZeGH$Y>_6bd~zHDUwG1_CKeO;IFU0hXhyc`woXrs$c*pO}&>RJ^3K4tPJKk~UJ2CzLx`~`OUl2Rd$e_N0;L!XPwkSU6)&N6e$kO0Sn zot9wXw_9}9G7j!3qpExCF)iD}#e)m7=J}51!`91CcJM5QzSA4#wYu&I zZpFb5A!HH?d%b_F+(w-7n6xOUcs+cp{7MoTk&aP+y#1Qw)jJu)O66aGl~HK(G&KRf zO6Qm*{!%V)GdJ;(cJ}Wpznj7Fu#ySoGdiBk-1!)^(}mdW=dJuY#X4-Klp1y?%~gK{ zM~X}Gh$jV)u@;nmp(prM&pi$kbFsKjy}StW<9VrB@@`2KAqLn8i%>HGI1(6?JP!65VXo`_$ zc^*q^)zMu*iW7^3e(mKm*EHjM#f#I`mS6>Bt?%1WTh>RYMy92JC&W=qJ)@;o6%+_q z{8mb>TxBM*3twLU_!4DXjma~nZX&a4zdgP&%_b8NDTae^`jS!FNWrV>B;6Tlu|=YY~oLs*IdD#3~gtU+~t}M!JeLR z@9`>|WX@9P!I)dXVDc8%0?XH-$ZuoS7w@DyNLQdR%+--y6>l!*8pEYjpo6UC!-ncx z^i8zkDjChdK4Fau_#cM*i=&UKZWQH%fo7>-H*k#bQnVm}&eW&T8vv@LUXJ-<_O`|} z?&?F4BkasUv>L8dO~~kdtl5m^Euw?JPZ}B-V2I%z-gzDWe=J~5qq1KICqOM(zE$YN z6K&R=YEIl(_kgphef7T(V}r42iY^Cj-?`;i$Pm}J!sQy`)NOFgKXD>nLL1n@5!VD& zJ;Sz}3PA@Yx$CH6vryGe!a_6u@8VO&_5l8i4xAga8y;!h?LW8YChVh_-eUY`WM!mJ za9a~c37Q`6Pf`CoAl+=5zo2vx1@oyQ2M9yWhb1TztXxILyKG_IiRdU#)W2SSsQHV( z`!{E6Tq$4}RXyb-7ik)Wny8h%!|<9}uB|^sW;*#`js|OLx{6aGM+MMV(x7HD{PO#VvtMrW<6WRoggW zA=L4h5Dq_ih*%$Z=Q9k;_>|1>0vQoz^Bv;a{G6+l{~I+~Ixpp)d+O#}vcVZJw`=sH zY!JcgpE&%DYkwXhzc)S!UKJXBSx2XT79<>Ugd#$#YC1G`hmm*LS9b0$`L0gqu$-XI zB#bKkgDp5Xa^H?X%h4ULYT2?%x`o+-qp8SP?mgrXuIZ#r!P!$));QQ4AF*O7G)?nC zy)K`yiCENxKu7jL%D`O0QA!gLXpP`3-ukE}4frGX?RQ=6KYGI^_VI%&-VT9pKH&?(;u{Uhw%2nkJz=0pgt)G(K0TyCM!pss?Cykxm?4* zMN*}D8Q8g;oSV1YyRv0Qem?wKO$kDW1 z7Y-@V8`!vbm2Si`7R|YlY!sl6r78*Q|M%=pAU92u8qsWy#*%WD3aqFgW;R^gV^pX< zyKB|ero4gX|F;G%qapqeQpt0-ymU7}ZrEu~{|hk>&1Nf1!czMQ@!b2oXfJh@b*KjZ zce9PX$H!`=Y|qo8VCByG8UDDZ@~~}f%uv@)tn#N+3dlo7+3!&Bi3YpwL9q#=bNHcV z3E0xK{k9dPpM#Aau6hwi7JAXmQqt||TRvqfWtV;cw>}yJH#m$*rN;02hvfy`RF;-E zwnjUmwfZ(L047AMl+TIAa8}Mlm%Ci|LbOUG()~2#V_+O2)2G62<`M-O;WFFx?un9M zPQS)JXSr-$WyDwgl~g?f@16r2H()`Ig}Z+KYztm4XGTZ*Db=y{_A>pEPTD@z&J~NU zhhO8NUG0HAGrScJYqZ0>nF5!pIlAwxK-9d4U=iCax-Kh*-El>EZ)XvJ>Ed?zNz7aJS$qP>jW$0#ck4huSblrSpz zF0T!sAuK&*M4P(xX$VMEPK|75XQO@uVqI`cqxUlwkU4I)qM&T2)OeF=@!0W+==(=y zY$85Yf!D;5nIwvS4g}5_>bN=sN_(?3u z-F@_oIVap`SibaQjMHuXJJLC|3XziWlPI~D@_Thwg5zl*6Nzvk{QFZ7HKWw!e3DS* zrsY<;5z(B~wG^T$cz8F0wQdJK;=k^g3>@yV{eW8`B8Y9P+=kiW)jEnpCRk)$cvp%F}8}8h+W6uT5 zgsXm%K-Zg+5*D`ccvt4=ZZOVDWT5x6oxL2;rn~r3PHwkCzrgy1_-!$m3!APx4%p%{p5hlIoDN;^RNq_vWNXq(oJo>>lsYRx z4OzQIvx5>BUZ+@W6G=*A{p)2oaSTR}6{Mg)CJG@`6=SC62tpjcZ^h04Uf6HxnXt%t z2<5V`3QjMonm^^XwYz3tx49Kut8SI5!em02{)l;#|0v~xpaOOuZuUw?N#+#8*CI{s#7q=wMW!|)*+rYJ+OUC0l5Nmn?#TBx9z?FgIqo@ z+fghXw@|;CSQ%o`0`VBEkl&H$>w-7pd7`$-*Xtw6%zC&psJA89?wZ!=?2m*Q5y1mQ zq#8q+i5FWXsD_LjwbMpr)d)EmMz2$GCPN#^_OoE}KqS)%Yxq0Ru0>-}JObk)KC!_d zky6tZwv+#EFZj&tO@Fwstf1bcTAYVuXi*~uYRKd|sO@vGa#GiCr=)f?9GU5cfZPYE zID;7F)feW`VUu?132y>odg*feR88Ep^*d+3zcLaOqpy>bKV>aGqmh{r_9R`Yq`X|J zcu8GwsrZ(e5OAC{3=kl;7CGpNoaQ<=|lUz3XT(6fxXhNwcc~r&(0k7?%a6$&vSN0+zb^eJXD^%V$Ter9xQv8>IycE%Ss|;eW6mg<_@>l# zO^MH)wdmNZ$C-Vdsv_8XFA5?Si;;$J!n42Q6;-yXZq#)R#1B3-+a%HpmblC_mfGsy zNt30$uYT2EX3;FLpQRXZfw*Ma)KvcZ%9vZ`4N^-KiFSQ0V{JaUIeqg@&;xq0*0iD; z(i25f4|4caCk5Z&{@NI6UZ1{MCddF1OHVx}+V z3J`+Q&Z4@YzkmeyAsn(d8*q}ztcL-^VKhTMSr?GqJNomtejJY{MVmxCa%ZZ(eMx7* z?}GRIwnOvjDkSx%XGaJlwH5ITy8}T)c6)?hl)msXqwFwFf02kDc|M}Z36{9{NRA7T zX^;FWj3(wqPfkyl1ZYej2bEoa)TW@sw*i1&V??P26nA`47E4tB$S~O{vw0pVD?8}; zkI*#x>Z-hXq9{Am#xUk(;YIA31ALOEXx?`c7BeF?10ocZ-dKj9>60hy%sql?WCPIH-a74kHw68OjbAs#$tX^U2Oz#fMPe zh_G)FP=(1;9tvDHWudKQ%ukmGg=T#IPD`pPmV`^fLm$md_=GD~I38HJPQpl-CV*vt zey{OJjfuKS3;C$f7}yF6L}Fm{McS8-1a{j~ z5O)Y*)k+MNWff7Bv?^6Z%^MJxO#>AaIY-(loi&cDS)9nxsDE( zr$<%JKuhkPp`>6tb}ZYbd){f2?tf1?1V~p|L&P3u2z$dwDBE)ep z@n$ElB=17;Mpow@1W^1~>@!!XS}#C4wNFfM2tBh^xT=#OL;PzVlU6*KQK?h@g0vN%`B0Q!S2C>fv8yD-3M`E_2G0pwjFb|yZ@#xrHiO>fz zBRy#PcIxNL^`T!peWF(KP3c^@R5IvSI*o54+acu8zI%NPX{FtqL`0y53`m4WfGr3#UdSEi_xTR_P|^ZYN=ie{e^FZ$P~W=p)xmdbm(&o za5&1eXyCsD5T_d{I&RdmF*il+JAcTr$sVYV{M)JZEo0akr~%Q7qWZzL7HtdF7BCrM{rZBeUMJ5ttBYvmhZgIC^@J) zwVisJrf8EQO>}VX|5$+6X1v|G#-rrd22nPfyWxw}tQge@{R@xS-!G?z?E}xEwj1|+ zEfy=o69`gS-B06nobj}aeiwJL+?UgiE_($bp?U++E!c+om?fo%_EAw^?sY5j`^;~@lJiwiSN^=pC@Y_f2_vpOtCc$P3087 zKLF0C@vv*Vw@npCiV6T&57ZWW%DrKzgKQt*uty!&XueVjDOQVpwf){`lhutk;D|KJ zukConnbav(UM5%hJmPay=HOVOtsRpl>rS?t?<&2D`}yeWp3L%A-(zP<#DYr!!L^S3 zOtBVTzvXHN$a(r#7hK#IE&VXrJq*`fOq4zyvQIz!b}_>9JdC0r7mSCkJ{Y$QGdEcw zZ-4$_A||_)zVtb!H($AhxmkWOu+i*ujq!J_(3==tjmt(et)>pfDP3X!!*H5%u)Zin zA$&IJ2}&1`)#j||upNS)p!ltu#TR~{?ERib5|-59oS3tu(d^Ih0!ytC*xd8DY{Qg> ztNprBpQ8Z&=de2I-FlX$Rv{OeOM;tj-ugGICOH{zI@*gJMys;r)yL;C`5zwq?pKLH z_ubUTu(1jU+^wISY#dzd-x5dCjD-ASTVEb0&Ry3tswCYT8q0znXY7vT0g20r7-)-gOwD8& z9!ZJZ1*%;FTgtoZ?JNxCNC0K|S+KoJE#YHgwE=%L@cbR8w?KG7JB=o*oKTuU^&dwt zmB#N?yI7f^z*2>|qIW8)kTl%3KNcTd6RnIX!@h+4P2I$Sl^2c8I)+j)S?k!B`SfE| z^g2Y`SC#1#JWTF_h$ic zt(5%CtoaF1?hSS^2_d6BUmVD=xIwhKTz~(O3G8kV4xcoyQ`}1uN``7o2G+8uhej(& zZNlQ^aT8gV_vgj*@7#JE`6#Kz^Ktb@=WRtkd;S}&O`$x{`Y>9a`?R?^eR2Iw`j*3{ z_I>s^Tu3;-d%rw<1~z6}>jnBYad5m3D&n;`(uW!Ra`{8{9PaY8mn9{yHnY&L#zQ6U z{`+MmrvrL00I;f6|a>ehl=N{7g>1DLkx`D1DTL;-e2;5Z8E5us3YGhjAm7<~wMvf1r z9bQmR{ARMWzRbcFvi7C54Y#^u~?7Rc5@bCfF|(JRXbS;{tVu?EMHH?%3QZ*LFP zdsx}{j)oJeUHQL$|0tTfyGf{E#5uK)^oElS_4~if!F(~pn&Ffzz1q{64E9GO*l*dx z#KcVn@59Zr*D*y<_#;mDmX7~Suu;xyv(fY3$1#?fzvnZUNp?M{3#{?)s9Mq8QUnCy z3&%f4kBo`=%B*(;mA+B9VudWsez}@l-K@J-6(#LjnYWd)@26vv^kO@UjDTUhwdA%{N9 zA+6j<#UqIpuF)!YmOsxNY0Q#_EQZHY4)wN+!f_s=QCm#E4kuCQHJtX&m5nE{qpv)m z9GPv#)5wNSj2;*6bWD#@Jned%v%Sp7Egyw5nWk>`Yfk63l7ZDf#1>aF^)PE5oUm{ zo+5SM&hOSF(9KCPOaKV<4A%b2?9)G7snCZ#kl)v9!M{hsp60-dEEjxstNJ0>>?@d^ zdtAqD#J`N`?jX93#KHOrcog)$L|g45CJk<7dSoGU_D3h$gi*UiqY(LBWxuG-5gZNk z1YTd|LepT-6ZyaZ{x-|lsqQucnVdg=lQMES5C+oIv#%n7KKm?)W%atKb<7h>&GLQ>kN*CJYp5n-&ptew zOp#J_8iVa5SXF0j&=$7Z%mzw6?S9qN`N!s+^^os_I-Bm(iJ9C(l%VXBKM0lWl(k^I zXl;E{a}aRO#+1=Jq7^7#1}i()zSca^`nE99{`9GSzI;xHL`MLrW zm*T}*+={ywcbB3mUc9&jLMiTE3KS@A#a&a}-8HyFu%Lmw`TWlN-9Ps1ImtP@`{bF- z%$=EgpPZWWbjujkvv}=Ki2S*ETJpePmk~UOVFgJ4VB6WL6FXy6H(z;IIeIDI)7p)p zQZ2e~B`Oi{`Dd!k6Lkg0YecrG@xmiWYpg~l)>fkhoL5DP@tTtc`_AAyzU!k{9&m?} zIpmg_wLR(jDCFhL3Bk)330uvg>@^{pUwcWe-v8WEOIn|>X)5Pqt@-s$NNU8jU}TbE z>$vw-zP@sMSOW`2=O;c^hSm3_LFjB=uni($()V5cQoXHX&*kLL&g7`X6daKc}i z2PK2U3kiIGUXNXjDPVjn`O>7|Z3{f-NiG^cKCu_|`!=x}O8Q&ecx71^s`sMJJ(J>C zBS$D^MqUTZjRHjl<2+)oEgEJdCeH1X7bGR7haMLW*a}x|hu>yP_=Y@~&+OUm?%X%* zWIO_aXssL{@L`@k|74Qi`CyPMB|rsZSIdKZ?{$5I&HTpLa=rvozIeeB zeT!2$rS$ved|!|uC;JbmrpMXfxFD=@R*(k`wOg}WsyIoA>X7FpKIHwKUz?v3x}saP z1=HyvZIimA!L%~AsS;;XkcUCD#xGVU*)>169}Pd1nNHPioyCbY54B61V4}B+YDvQt z4nJDHg(n)Bir7i6-X7AgWmR@^cK<1z{Zy1Hw6**Y=@GH)G!NNk$a%1m%;&o-Ryqn)V9y*2 zgB0qcIXDQ&6Jbp+ts4njuz&d3Oj=JDVV>Xb?b)#M`ZiGk_e`slZ?{MO!#}wMDnQ^}JA}`}b+|-H`)cM`Ef$($(_)g<-sC7Rq?A9Mo2`Gn z=4s>oJ;jtL``QDFHDC}$#mY>D!T5RzmtbUMWKxmpa9h)L5oW)*BTPIQR=$`_?qB%M zly$_Z`9Ur9gIf8#L6KckHYNUt+o_|Nlv$$!Ksj5dww@bunKV$U(ljRzXH*{T8jujF z_Mj7S08weF3_Q3>mpdX6uPqS6o&JfM0CBKP{e5IJ?P8gY;cNPPyB|5~Au@7|$%nkP z&1w2+LxBc`({VXmPaDr!F_}9XL8glVg090RLPn$jW4Vczj4Fle79R{sJ;@DMRZ6GdzxV4MOT;HO zkhRD6N4)KSBP_M-vWlS*UGvhnQkui5Pk!CEXl;sBRYJDpg_J$_KdO&95ol(WBKLnU zo&(Eglvq>jYB|LdW5!5OXwTV-n0K~j*TcW#b!rz%N4R%?|JD^HUa_NJsZ82}BS$z) zbXeH1Q^t2VuM;9g)uhXzoV2Dx`)?&Xwo3PBeUDb4jj#6mB0sqdJ4tJl0GjmLI1;;z zSpR}xviolYtneolv1yL{Kfp=>+LkpGv5|yTM;F9BLBWtL`+cf`W_$C7+nGjwP2g_S z?JquoB=v0Ws`9$0*~lK%)z}$xNSkHPYTpROgg)PFjwp;)wb_P6R(%zYT>q*17>FHFlR2r%|8qXI`q=W=xcw1h<@nT+Xl6jxUK z92}P0L>_zYBF*G<=52qMEyyMQbBO-@g$GtfAmd)k%cmY*S6By~9I}b-n`jr#k{2#u zCJY=ZeTb9DN5zTTf@3#CWu(teuCBR6S$#~L?f5hu9kIe)rCB`1?0@B;(dfIx!dm7Q z9d0vRu(c=OGZm~BckU?~tY5{>Ho?j~4TBkKP&%D{jkO9wNiYs+PS*>rz^Rc<0TghS z4JHlIpQ?2*#q$CAD~-xa$gz8~SLdxtgco;WbRBkyZJ7@% z8}kdBCh*U^s7{EP+7@e<`!%Q)UR82guncp~+Vd+z;5zM{8mxqP<-=o~llFW(GVz5q zUElJGh(k484f_*57^agCgYp|nr@q0zs`SpMeYFQmfj{`7cya;Z8*x>Rs^3^}fWD`*v7twfu&x3N;ER|HA^{4B{05I0MTuVdzaqUB$yJLtHl*E8`wpRWgY@ zzm6G9EzuS#nW!Jrd@|m)jOipSIKNs~=mmVT` zArUX64!$-lS5~Tzw;da0jsO>ugW}G<6G_tC2ooFzU;@N=>fEtnf(4Gb26x=b8AOUP zLx2DNZ9OtR5LpF&E-Np`>O)OewQ1;H9%G5_Ni>O72{E=<{2TzT3EXgd7P7tn<=~R_ zA+echDFAl(91roBh$7=Hy(0ySD{$CaYGhIXk+NS+R8Hn(0FG9-t-b~IB;~a$T}Oe( z)>MV{Ku^($R#JIHm42Bz<GjGRbuY%v>j`#O3+m3eEuuk{Hs@wZ%@U_CqS zZTik$&Ugmc3PtkDB|gBQW1k(9kiCfs_s422Hpb*b?yr-u*pKbI3(>H2=N^mGi+Nk> zyQD|={-m-Im#ot1!E{L~-2E_@IR6Jo1+(f;Gx5#t<6_r9>P66AEV!aU&dNx;T(@EJ zK?#fLr_8`XY>{F;i9Btp$<@pf@5F^BBd5g8DM1>KmK{-(M6*wCh{f63&;D$|97AK8 zxcTAn8B@@T>WNx!HU;B;aGHSA!wq^AdaAZ*rgV=M?UrEvaETNi|7%xB73tW`!tI*d znz4K=w88m z{nB`P?sHiL}!n<%=-+sM~a=)EG+p|EC66p{Yl_gEw* zp1XSa^6}#n6E61lB5rqsKq$D$>6nDb5)%^>bTug>R0F)_=+C`G((BgLI}WZRCDHsh zQNc9nT;|%hO9gV@R*N-oyb z)7ylWNqpOoMEKrsbOkf3dWbv)+^$6j(yL3t+aQgNet5-UKk+)AZbbWI$aS=}^^JiK z=bJs89d{Nw?u6h`&Oq3)RzCdX6KpPu7~p>)oJaeIctRZULzb5Nn(V9%O4{kFcvle! zzXI4dEIT(LcE&}2U$}+5*7`vO?e9sVIgSCk5>t|S-U;= zjp0NAe)^X8M-j0i$r*T1i6;}UGc-Q8fwz2$>sYNbSSWmN1mE{1jpHS!%IP57e8D%Q zG&ZFcWvZu@jY~5|;t=4jVP2`8pLEP$2CU4PS)XgVReY_pH$PHr+Wd`AeeuMrZmv>~ zr6`v3aU@_Lg6K_SrUil4#QHN|z2M&c*|x{EP87XlQ?uydM?8P(jj*}skwlEzW6V)S zyr}^rMo@S4e1&59{S_Pl@{MnSkixg0Jvj8wD`-LFk`99vc zNmwcdpQlXS`>xreD!o>%>`i>8Swn(678#XRd6Oo3`~%@t%+Z)``d^| zq4*qVSfJ;~F514IdA*Q=xiY&qHt}d#m6*VcpK6Z{%PHD}&QMRp4>lk4#85 z7|P5UJy6cwrOV)N-~lS6QT%yXTt5FjCUe0AMF<$5D5LVPmXZB@%nY9ZZjk0Y;C#A~ zVZ$ZTjXL3PA`DS^E>*z|3EKEbH z#6*^O;(olFkc%dr$l!lmz=I!!BRBZqmx&p=c(IJMv;jDa!C(jG=r8W=blaS{;}UYh z#Ubt_8zdXOB>1uN=*S+#?VsMc4{37YOiqYk7}fh?Gm%~Ad&O9lgmtcASp8ioT9)=( z2|aP}*;u#aC!3!Szpzgnxy|IA>k`*z6OpLq!htb{af?x;y- zi6T}B7U_6gmlI5lHNx-YD#pT^%$DnY$1t#-dPVK}6M%u<)Sj1D^1YL}(iSuY!xr3$ z=7%$b62xewT#ib~xeHx{AFBddEfqYe_y?5PTQ8DI4StJKm(RP~;@MKN8m1pz?Q4k- z*35hi1X(+Qr!MQtZF6{mK644}u6m(!{A!z^?Ic4zLIvUvFCx$$z2^cSefcE^>@A$) zv`c+1Jy;)(7IXK&h1y)t&kJzxq)XQa^V|B`c`UDAMaT_-b1IO;s+!(>#R3?3@ zML$MSjjPN+vr63|#)*%3Be}Kb()RH0RBjVu=hoiE7oC!vobNLD?Y!=7tDFQqjhI?h!e8wXYWQhO z{s8e2tjumE+@Hp-K6yj2Y_2CNY@d5Qr6R|m3J45W<>=VE-A|365J~8z5fH|zBzPq2p%Si?#@>(B8 zs6kH~!}gM+NzT5!4SY<)rv{ebo4L|ElCO`Co$(Cv2IAWxx8=*OHzpg-FD+#ML=a+v ztS}_Q-uvG0mzI>YA6K@yTI?dV_WYkii;DVdYin~LS6C;no1xDD(Cw*rtl9@WQdu_CfjQ<9DBKcSB$J6;KU;f)TIOCzl_+q7Nw>!!czao%hLGp5?c~8iz1pW24Qc6ZWjwtBN;nKF-%DzJNXYm2!*O!;;8 z4gNlD;lCp?>%h4)(9U5<(;6cO=#14Z z!=qGLD7YQI!X5~j<1`;R=zf@{j4)A$Ue|?V4jF^l{#A|+S}N>+qDK0l>+fx-WC`eK zhXs(qB%*hxoF?t)G-~oQJ>bWRKGh0Q-;>1}z|&D$PQXLDSrwr9>t-8Nn>zYxT%41R z?RY0L<0$ye{#d%G?@`K4W21)TV|{TZCcd!CVJ!w>zju9;%UUaJkN`UL&RS-;NIl(1 z*pUq6Fj2XtbvLWJ8P3=<|pR%>O`}tcq^PObJ<#1rNAS#o3j>HpENnQ*|RxS9~ zjqJvYPY_Q`1+`78TClLVcs~;SjGDQU4SKMH@GRQum}$ik2Eu!2#*X<6&nKG+3FQozYzAJ9S9PH7GsIigh@|p{PM@M^O~_1x9 z@m`bVpQ(LF_&T14g}s*5)>!4@WS63}Yf;(@;I>#CtZ871=Uq?1mn2@tQ!@JEu|wM7 z01oQcf1NxyW@Fj%&SDX&K6~(^0YK9)OgEsJtK}mFtQRR@X~mZLI?{bsVOxM?u}Zwh zjJEp%gSogHJ=_5VVLh>=# znnh%Zzm>$eq&`&`o={bYo~x~W2{~ClBdJEVao5fciR~Ze*NJSU8W4h2mY0K`YlJS&c_$@?-o+o$J4denpFcmZuG^CEoD9Ev%6Sz@N>*K*5 zCNCu#=RI#8SG=k9K%oZZbBjUTexr*$;j52-kT$S8 z$n-;&{;fD{C;4vR1hSu9VsT=7E9D;2{(1fZc1#02(K=Bw@_I%EVIj3d{NHs@rY||} z@yyxVzw29S_2W<|)MhzACi$Xh>xNTb`L-@ek*NxWOM90!QcBu-P!9tgpV=HLAbH@{ zH8@&$<`KSjF;L7&zLnwXe(7@@tx;ip^LwGCv7Uc2c;7*^qx{0KY6=4d29)05=}U9%NRQh%9AwGAtmd=Y;8F2sCBs5l_6l)NzP}o%sQtJs z;|(K>+>vv}hAsTpwMGF}a_g1BDH1e>aQ{Sv5j?3y)qGJUEFY_&W~!9?l( zEhXr9gl$?p!7;$e7*No9F}E*xX-bvTUiTOW+}Fr4*3!+Ga^Q-mG>)JN20x!jxoDI` z^Kat*z;GWL)slRg))NtDyP}w<1YQ+ZcsZDmcD^9vIdERHXf$;l_C$E0(Ggy(&T@*7?y6_vi4>cfz+_>pKxMLcV(^m+6(AP)*Nml?j)xb~oS3zi?ja zAevPflXv!YqGkE{$)9ig1;Xj11M+m0CcwX&n`GEUoHhL zrNv4@;|%F+Q_FdXe60=jlRdZF)+xp{+P;n!T*)7~&+-(G9)6?#7qGp|)vX+Zc5`3t z0HGLKZ7@&M#7bekP@v zww0RV(7zo*tOwkNQ8=?xXp3n`?yZ&uER`Tg^VsfKe^dugskF0Kj zR9iK%QF0v`CFiAhM}brkl>wQyLMLN|T)=+J+ws3k!F%b0D!D zET4uqIC*Bc0*~rjv3_!bZ(5HjaR){azUgc~QU{rYm`+nXEY<9(3&gFT)p?bDg}i9{USY2hPrQ?zK(Up2hx z+w^2Xojl(X7RH!wxw?}?e#MHo9VY2tEz%ufsO6F_5DwGdGx5Y~h;;(0K%?+(^1db~ zCa%rR70uYj*WdRsfE;J;)X4?>RZX2@$82z(bUrC5BWLM1Q15E}HxGo9OZTtF zfa*|*-iPkv%(nF2r1YMBjdDt3y>Dap+v$`Y0x`sjJ8na!3*-HfX_AYMJmIZ%92>6* z#vQnrA|Wu@Xszui(Cz-NTVcw`Qb)rL7_t1FkU^&QyzQ>Vd+l~W!q46IV1Lw2?yGqd z&P~xbjk3pedPo3Eu{4p}`mfF72%@R>*(%e7 zgF>)FHK()}_gA3ggXz`M*(6wT`!eHKY~xiRpHhA@_!f@FsE|nzc0)lq2`EG-7yiqW zuJ}>BxV9x_Xw+Svy~Ur>z!A|n1BEBQt=sxw*fT(ES<&FTJ4NhrCMA`JF?N#tf^GSi zt6V(%UK+IuDpSA3{Bg{xokd){e?Qw0>~2gIm6Ef{FUT1mwH%1ZT3a*SSc&EiI;}f~ ztRkh==x7ta9^B=L`5lf_0apDzj4Ku}X8o+4?HF&0Hm=v_1`Ime=T==Xo*qTVy|^Rq z7t}b&O}T79yNkZ5s&UFRF5#G6)WJ~etvFKig~vQn_|`74h+tEMiTwQ;eF&e$41Vq_ z6qG#U??g5;MQRA(qtnR_GY9MQm<-;gfr0J91I>Kk<7v}Mqw@!G1HohHF&H+`bUjl; zWmMsO^II5sEEc-L?_7V+EH7UoRVOQ}tGSp2Nkys8jST?qAIAd^3j-04eWb3h+Vo-= zXHt>+RV%07@LO=z;%RWk^xf$KK=Sp02q@JS_+PIiqVGr=!_U4Wwq= z+4t;+^gT}j_j0WO3Ab!jj#K_$+4nQb?$|eHI?TG-k&b(-?xM@asmj)NOa2Y|-ta*Q zPkHx0n&Q9%|VM|3dTX?TZywHP0*;v5v}xFP)chhROT- zC7#2jSLFgA87HTt)T)-JTz6$wH%i?7!~Bm7#7mfkC(lfr{#VfB_){#;muAH(eZ+V} zCjJ+T6OhIAwIQRoS#ezTkUtk4(Kxc#dTH2d%4d&LW@ zdy6AH^t!vww0@rT^3QMVlfTkMQ|X|_in%_6|W1Kx?qmI!C^`(C9Lmue8IZBDZuMHp!VKF0fxul!a$Ci z+M@F_j#8q~qtBf}9qNF75nZ-0KiXka%P3-6QT1%Z2f* zr^?jYUme;=O*kY(xVNa~C6}e@N19x-z33}X6L5eu^J&ab|GRU7uivh3U?A`jutFZV z?yEw8{KX2>%))XWBoOer4Vw0yB$dGXa=+vTULep&QH~L!ovx%NuR8vSoPZB+gQDk z&$?oT+4J~FrmK;<#?j$1|BNwyI5$94)cEP2<^J(CD}1S#=Dw2U_fT~%U7~WG+a>PV zN{#z*295)`Ow^Ru@s2F=b3dmJzlDj+8cSUt*@=#xCH>p<{g%jrx>rm-KLV7w>~t@J zCCws$$;q(OK!4}Lz8S2$353&Xz+lJmTqj+uf_lS-tU?oa@0Ch#9u{d#p2W0eyZs>w zJVT%;#n<61C3k;=V-I@;3iEFRSQTdfd8ugB+OA_y z{5h7&R=YA;TC@AzF2MTpGtJrK$$jzs2Hfx}EH=x8a;u5VZ;`s!qOk&=x9r>ih3I+1 z71;tn)}L2oUr%*!7}Yolb{O~rn(Ur1y)af^AuDas9Ms_*EX{M&ZCAm8PiMyv$ffh? zg?@#Z=ix_QX|NX1v?my=GILsU<-TV@p;>sNW6dT-QL2NUCV(ToCd0|0Q|L?65+1Zn z)HCErwT4quPQia`yH8N9$H?I{Z7zQGd0po=;Nd7aWj;eBaPdHi0Ue*~!{^pKOAO$OK z66ga<1GL}G;5m_G3MhzQO<&{&CeAq`HZD@LV$a6h8UwVP3WG<0B9B`BuBW+g?N87N zGm$+P%~xx7O$xm(pvg!xB?H#IA*;f5F0F_CP3wMjZPB%$Y~Xkc z@|pttYwF-kOfe}Vwg2u586|E!xlF!7y!>qIRoIQ!t@?*zjXL9?SZZE0oQ#RD#jm#Q ze(~C>gFw$eKV}9)x#BsGY9ya9nAQWK9i#xCAjJH6838G<*6PB7h+w731$>aVd$m8& zakK28{#(d(Zf>FdetyN!&~UVEAdaf-b{z`T)583VJ% zfq~npYz7vR^V3&9&1quPtBl*Nlr%IZj^@e<5`56A)O8T*Zu#i>xBQd+@S794e;3 znmC(%>T?Hxz{oq&P!9xtQTsl(WrA+kT@cT(k9c6jV_-nOp4P3p@nF2(?AspNqsZ+o zIZNkGk9VN4Y(AtdUBr})Oi+b~U^T05Y0!IuE`2g{!!OURTLg5nxEfLtTjwB@HhW>v zWw+M4thdWUI^lmN58Pqz$7033L@83f&Cf-a%<`c0QjK%1?HtWhI4qH9l`N8w^Trwt z-u|2CUGs>xyqavBn`JLax^#5jWr&li?Te*B=0A^|jWP)yrlz-xqu$>80 ze)dxrtEBV!uqM}VR5V;T=sTQ3{Gt*!Yjj5N-}|7|I>LV*9O)ik7{Yv~aV4yLp3uBex^dT%`mhEjcYw=wU zC|@-eGx^+~)vwTP3Fe^xoItRr;i-ULzG85eAeMdseyPZAQ|AlK7QcgC zs&H1Bz{CY08d(CTyJ-Mvf~hq%U+WaiLRs2P{oCoU4XSy?3Yo+AS2gBe6qE{RVkKfdO-Dm^KcosP9LB8+VSg;L(ZKep@42S!sXQT-4wCED=0VJbGR1n4ObEpZp1* zU0%!ftKB@Eq$CoE$#y<~uStSWkFkVKTkFbcz~A!9*ozCkV+x9aTKmdJ4~F3wbR|@o z6OHc#&fJjgW{LR=2d*}yp9>OnYHbBpVi?&lbIwCw_A>FGL*nxLiFri(oA|DItc6?@ z^y7*eBDKRX3w70?HM!TWAtoMkD2Wjdv#rp0av!YT6-Qk7QoFRNmR*=Eqq$cuXVvJZ zYd_5_rvUT_Q8m~JSwlTflb>iq$d@IqMq_9}o2P7p#M}p)z1F6<2@gYxR8~Vlx6F|1 zrLy#W>;a6nv6elZcs-pX(O+$URfp)LFyn|gm1o#Gl9Wt?*> zU$7CDhAf44#^^AfrtoYEyzJguURSmI=-8_}mCj0Ic`$pQx)eIAnqdi?n5TA;`Tjj} z%}LnCgka{DpZ^ApH939AFAQ?%FCdKKrBpV+~nciALN7 zq0A+HO|dV#PcceY{ZNy^f|`^#()d@Kn9EE*Q!X<6Gp)Y;?w&*wv0P>abs%m-bj8fx z>e13|O+ra|?i=tox4puX(c-!RndKjXQIn5Z2``ZhULD&8CijCdBzSnquF*Pv$O&m{ z2iNc0cICb*5SRa%gUk* zX|Q?icN7`50;=E11#3t=UOzvcA(k|tM$O;MZEg2kVaKmhd(*Ds1K~Q1$>uJ@G#~^J zBCgfwwDtq=72m}B35HC7)QlZgsG8d7fJetmk>{gUY@>ao0Jr;H13p1P@#{&KW{X|o z*|NWPhIvJMA<%Tm$7{k9&6dlu@^ff00#5W2BN+WL7JWo6S3;Zj2h{VVA6E>)mX+^Y{ZbKYhE;es7c z;Kk%olMv^kIxL!#S>@G;N ziQtumf>qO3uegV0iX!I~H_SGTZxBH{?Tc5-G%>;+NVMoRDKqor*}C4s^7I14g-VzQ z3ddU7<@RTaGW8v!$5^{PT{4{TE{UNwzMILJphM4VT3GdIe4HL(`sR`)Jkw%JdSa9D z!f3)H?w7DEV_Ft)VDuvm&7@T+iHT&WZ+T%{lxK3ewVrQty)@{nS*8$m7; zHjP(6lHV{=2c6k1A)Q9fGL0+JPnHeWcJQ zb$DMyLU#7s{>&e*Qf~ugKcrnt{E$IKP6iZoRKRLSI4C5Y;fKr1H1k!qJZV{hT?Iz7|t^(e``4q7jPCzN~GL}6c5ez+fr&6?wAZhHBNC7Ai}vh zTe|uW;;7BLyjg+5p@E3tlBbT{+f^Xv)uW)31>4nPO}31hS;*%paw~)7tI@E%v%y%}RwD?VpEt zt4v%6|8an&;MJGRUabebsVbb*(E!yT%kYe92l^f8C%Hm$Rg2-jxp#em4##21j5>Tz zBx7nY{c`Kk=Ak#`-Ymprxw}1WA}f!N*t2H-=r|O@l^qo=ZPAi}KZjkM_X3qqVJx3s zl3d*VSzzYzRF)k;H*2+GM^x2c4{!DjbR=yu3V%tN9A!CYOCP0~23{sJDw$wMcB-bqiUHT&wrR>fBMI z5X?cQQ$qVl-R#YqBw??U#j?5CT`~AM6scmpA_(z!7|VdsDa4(e9tRu^F}==$&uA=` zq-eYsV|a<(t~LB#9iQpHw#*fQ-_uY0AwQ+IJk_a! z-uOyc>Au5ymz8?uw8WO3_7R&2fO67!N1FgQh#*zjR9CEi?{fom`ba%I__p-wF6S-! zD}0iFcpdVrRywpK$eQIk=qkCYvJaP-h%I8$OzOdcd*#Sp^tZbA_lPlj#98za)AM;n zFe(3K_l$#JWyR*3J!;We_$(d7)p+d9Kbh%P5!HC{^SML9GmMQ6v@0x)3gPCamoG84 z%Due;?)Pw`m&Dn=Lgtqd@V<@#Y5{Wa&1^;-$?ZEKn`FvPAIV9Mb~jE=%%4wo#kYm}}>}h-=B= zy(DPGMe(jtTxzT4v!kv5Lnkpn7ZJH|n4CaU<%N+5sN}a80B#UC@{%y>!O8p*#376HUF#;mopAz;voXI0( zF3OKfm4Y-zTAAp8zqcSJ32uhL9-VHFZ>!X-%u?8xSvKO_sv?@5NH+2!j8uU)NS+8E zqf(eC|ssehPQUi8uUS|YQ97k)p1 z7zl|B8Lz$K{3pTmyLsb>HOzB8S;x6;JMI|dVQOYZeYL+7R#(;W@UBt>MDEzpfo!ia zb06bhObtD_BQ~)Ngm2T}Qdt$xeYwT+y4mPWH86>z;u1{_j5&L8e4<5=a#0%IgfJTJBZwouNLOg<nMQnJXDUxN?dr$55Y<3ahiz5&sRN5SebF?gI6z z3Sw({t!2*cPL0^dPRa^-y5keBM)R?~D(K8%DtWqch5Wjl3$a}d{ap*11DI~|wq=l} z97-VzDejKVtg2a|TvJ1{;U$)!ja1Aj@3*dYKD2{Ep)s1HZs9gI#7B-cKnEvVz``HO z)_19`40-blU-`4BoFZTu?-!!X%f4SYZ z3Am{RmRz{0adGoVPJrCstnE~X7~LI5q+O|-g-AE!j>Z}*tzM_F_+R8z-@p0ox0vrM zuqL+XuFd5Cuz>27(?zA==}51P4eMY3h&VFd;G=bW7IFz?C5oRimBZEvj$nQ!UwW2i z6cxWi>79j!P%?VnPLAU;{X%F5)u%gkOKULPbO5Gxe`S>dNv~FZwJCg5@mH$4DWd|(SP2?5JZ z3bCiHU%kXa>p1;LYY5Qgms)TXUu)Q?_bAt@aBg9<2}+d3a2sSkf8fTO|A zXiVgSpX>4}zG~LO`~Ca-n$MJng^LvggHFTt`sEBAt)7{EiS$$q9@|~jMKj-YWrsfY zlE~6TFMU7`r`gv0`-KF+#^fcu9Cj)uqUp>Z=)IR+BrIrk(QoP;C5GZHF4CZqvC>Na zw+E4r6UCK%wf+3Z>8LJRZtI0hQAP&_ptLk?$%YG~66=IR`VYj4G_r5g{|7q!t;n4+ zFcxFooHzorUqazaU$*@Q+bNW)eUWb5pkcd8;R0$eLRz@82leL#8GH6j=(( z3?(hThtAoq3>!L{k@rS9xcugED0Pk8oC5(nm!=~DHNuV(o2-#yBZTc&KE+55pi({| z-T#*Ay&C#ge~j%&;ZJxAPXS($vPaM?BLCSm#lkzkP+z;!Fa0WV)xjVB;dC>}EvvC6 z-UP`Y8(UyY)vYpzFFUm@v165o_N}=$oyD_^AhY{mSTd^(>Vv}Yz??b$}Z#p7$kqiY3q?=U+1C?l;$l#@FkYB3fA>6 z7s=xK;)UHy0%pO2{cTxE<&_F`ORx-UqR(>%k$+>FQo&P)KpW0a3bgKj_U<>CzbNP< zWi+H;N@kL)3M|Sn@K+lt(phJzKBx3(@Clbh(LEc*^>9lD&<9|7?7tEwjs@FVmij_`Y>1BhB+bBB} z`X5KrPsee!!K@DaXS_8h)6sggejGU!QvH}QDwW}#2U*x1#V^rMlPEt1$@IOJsZ;ui z7C_@Qe1oeV-fW?OZbpeg@C88Vr^eOWi?(Dj%iI*)SSn^k=ukrv;6sT84!j2B*f zCki7=VUzR0yYMQns~BQWW7RsMO=GE^8p}RFE$<4Tb%bXwvkTOg_WVs?(>(g&g6|(I z@L7a56uHPJ>;oU8wMm1RZ(h*zfcl(WEmrE;3yo<2rndP*>9voV=<86z*XQ4efP*{J zNKQ0uWoIgU?039>No*bJ46`pS12<7(l4aR{D(+O+ zs)bedj#{8HDd$BlV_$&kdMrF`j#BQ-X;Yo?OLfg4Hb+NlOLpsaaiuhDUf~>cb2}t# zH$Qt>ZoU%o^Y4ZhLd+Y48pxw&9iAHN_MR&}>Xfn84+$CA<=LBX#VF-i_$Z0D*Y#s| z=VupiI-JC~w6%9;kyCTNS1f!s$%7Q*8Gfyn6;DaVLyHeoUePjYO(0fS08eE5{@)?7 zgG4;960+EU%Fp;*XeurnqTMS4ybjbj#9~ex^+x`p8(@QVFq$f{f=3aZT}4MJQo?b{x+Jl`FI8 zuQ>GuEjy{7kk^t?G3Kw$(i9ito@dc#G&g=X`k>Fkz}oVekwA%o*7fq__rX8Gh^+d> zEXL@3h!s?@cxxvzrJW+5yC-S-$Uah=rHH7F&n!y(olsVU)`?AW&n%5(ABQe&B3g@4 zo@Vr}a-^($MCrUn#v_XNF>dCJGs8!DQDWs4&Immd6oGwVYeAKS@~g763OD1dYKKs2@%YL^jB}mPoLz&xxe429GHH% zu9VFq8TmF--fKbbrABWE*{>I-(k~v9_4L0vi1p_)Qd0n`z-e_~fmEvL{&RHauaW2X zEs7f9QSS*%vZU#la7!Mq(Uxfn_r2=A1Eh%Fe*ex!wex{ns^c}ro5t_2q+g2aimVtb z#rqH?HAN$brrR$Q4Nx#7i$cyH*bTIa-b0$iP$d<9{P&;#LJ{L%eA0Uqquj}HR}@x? za}@d?T`uC^l{l#`-lF3F@cCEe5i}=sH6w++g~Rk#6DF1S9#upZ(@o=F0tfe9G~-1S z`==mU-NBc1x_OCc|50`!Ng`fts41b~$0zMc1u#iYhFvfndEypz?YxGwUC1Yl6LUeC zuwSuKDN+VmX_U~_;zNnaH08;MT|T*hx*xcH6cQxMd%U~fN%ly?hrh$-CY;%?#}K! z@AqH34>+Ttd(F9K)u^g_3_=J56DER6$ah;MF2_xT=bs+<@A~Ep#e^8r<&GCHEdTj8 z&J0uy3OQemyiO*vaT2n-+SZp@&NxrT?#!J_dISqTcd~~|vi5p6e#iZ~x#MvOQi14~ zJv4&HO<;KGzk|ONRs$(N5cwEonLS1D6b{=Q@0%zz3AKS5I9~LsP(S3|uM@TlZ zfU*)-Sw(wZL+$3prVs4DR~7Vm5idWy#u9Cr9a^xE{NtpwzaRuwq#(*px@zgqzgrl8 z>re&0seFDBNNERR3Ldv>q;ebfNV_x#w4#(0)eTYo zYRG?q1_XpCA>@}|UCVejzs1oYXSw6{{M*Pb{hi3jVfzOWProANDUKLV!U_=Adjs%I zltEre+u2J(2Gnn3Tgp6!KCZtP(HE-dvMo@gS>XyTM}w$*!RgFC^#83Hh%gM{a0wy< zbOJfx=1{ZT5D2iVOr+d@04vYtgl5GI2VgpmF_0a0T#HbDFq2_@Yo zfca+jTCDqq(d+S@cP8Ae#GvTODH9zd6$1jCuzk1??AZrGuv3GHYJzOf{5I(aTC9GK9q_vZNTy;|7Mi4 zQ)r|trNDB}k{Q^~f3m-Cefur+17>E+p;DhAC*oWifzcLnI`EsDe%Trn|8pvoz@Kto zlz8uyaGuX}r~Kr=tV|a2v07#GOFE%K8vp$!T>0-O{a?*jslKvjMXCyOs2Objvw!d_uP zK>Q53N&Ju}rZrEcQ*WRUzQGQY5UmkIW!Roh4z&jA%bM0HR}AfCS7^em)h=7-)f zXgK<&&=>%^gS*jxF$OGG7CV&t$1X~$k1EDd0dUa%1<=-&OiGG5?;4{}xpI0YK8$Xnc0GtR6+3~}f0r11QfWYGh8vr8& zAjDe%V)cpm8hiZE$U*?5l$6b0e*n2;X%I!WQs}xbVyFE3qTeTo1Sl|{jE=<)2Vwl# z>G$@lk4g7l1l_Nt{*O@#WK%M}Oef1N7W1oQi{XFV7BFATlu#P*Uk?){fF4tLK8}b1 zL=5i&H|g(et%WFoGoiWl>;HvEHi6pyCdm%9F}ZP{H0iBG za_EA9qrq)x{|Vxf;_oVM z8>$!VA1-{sMm9EbgS%>*iOFIu%39hbZf)E=c!+Hpv_s>=d{!s>!^Wm2PV4AJzs5{~ zR^WH{C#23!WM+E-`@54vH?X%-u1e~c(h^)I)9!*pk&#faG7pIRun^H9|EJyOeFj4-^@DRNTnz`oUy6t=}COB6_eBDUI${;{9gN>tKauEMbAxh z5;Oimn;{^=xPEGk@y3I*l3?@Z60jE`fnBU22+&`JKUvbDqU!2TS~3MxPKpAfPS;nS zG}06wA-=K4we!jF-m@9rJ=-w82$#@?HCscN)gx5qC~_9?di0eU=LPy zS^qO)U-@V&F)^v=R|Ennl+Wn9Pt9d|>N$@Ge+njJFW((4-Ig6MO($`FyPBSMB&Pt? zJiNeHU2tmy@U92<^(&fNPuE9=lj4@AC!V*9Vm~@<-$F2<{o0rt-^0cC7dJ6yd@b6d zvnu)jZCh)p3p55^*U*oPRM1~$2_YcDKiw#h`kx^pVbd3oURr(!D8zhDFDayaGXZ); z-W2X?5Z@XfqET|!?Io(*Q-s<{h6IRn@Yn$bWaOS2jgc;i48*t?&g`|$MI3)H8%~mi zf9gB(YpmZ5?7q3Ta-PNOa~3v*Ha?$2Pf0y@%mSQ+q30dTb3?wCV`Ug3)Me7rBX*$U zuEu%O;+c4TMV$`6ge2#acL%#W?&XU1;Ia6zMwH`f| zAOQn#Rb`JvR2G|?l$55&Yg!^uFvvmMKVNF;vuy%a7jPK~Z@p3qnhUCmeJ+uNfPfO7 zK97zQG>C7gky<6EnbfHLX*1()j>??ZBupzCq{V3O00ALP4(C-;W3gP^%@C`DLan!N zIGcQa@43@9PW}zp=h`x-hugraK40#nlB}&WvjTe@7cO0=BqggxVY97UGUZfowOg(9 zZ%dT7Fe8S(zBau(jsKAeQH>rSqxpN5g0JnT@i<2DqhIcd7w2*4x%!^=w4wr+t=^He z3h||@qphh@yCJWsY300$kiY(ZO#R*EEm$2h6@iMb{IBZL+S1z^N6yK5?dShu31l?X zmMScWUQA@b5jWwd;4qx^_$K68X#mmhpUN>wabA1-PN`dNEZ@5QfJf`@#^?FTd(R-$ zoB{&Kj0bDdLs6n{DUx+1EcY6;|Hl6! zOtf9IvaFQ>5b}YG2@<#q;_P==K{5bd7h+$AR53{ed5eKyIE&ajDhdN)!Y>trwF-`Z ztAm8t*mQeG&4I;OkH7F<+1A}j6lv}3InJbv`k~D8pTY|viSvy>Y(wz&->o+x)DA8r zKU!UXle|e+n&egrvA6k_1wA(e67KeE#9};vhES7{PMp%0RmN? zIOpp|n#s4{A;KSzK0`qKg^fnBTi{$>%O<6G0_hNbc96LqC%X{xjd@oHsc zdz;IqTIo!{iEHuL5##M*G<3#HxDkYP!xjuAR0VR<>d`;Yv^0_gAohO>Ua*yiv2f$G z!7Q#FFlAoMGB)xxnqJ+ng_4O^qnMjUV%sY4DsfQs(YK4OfNEESEx z2(trI=+q!OaA&$k&aD1m*_K!~Z}QdPNkq8>wnP7@s^+NEs!>TSXF+g0QGO^ggQ- z=}CJew$sl^dBz_X(#X}T2IDyM)}JnaTKT*%yRAGg(~g_hcdF$dn%iBjgMtdM;GBye zAL%{z_uKpma;`>Oc(0|_AOBDM5by^Si-bs0hZ0dUo$7gM09A?@I(KFt^hQOr(nB_j ztTMlBe#X%Y!tcu1iZ!mns1#OC$ah}sX%j+~tHDj(coreHI$-=NJG8|G-7Q`opOQpr z-baww^2%i3??9t=VpLm^+hu$Z-J;32q-06TMJE01zvg+Rk!F7^)B7hzJaGy+bRMS1 zM9@f!O}dh_m=mf+4zZ*xZ%2*P%8oS+HLrp9q=*0~kTZ31%DnK?jYu?!t1>#6iEi#T z+;}Xyi2vXjTDU6yATpWVj8Y(C15NRFpcFUC9fhzEZF2d4 ztHZHn)ta5iFkjiqx)J@*3&J2St{{$1aZ2ZnhethIyP!skE@l#QHf+yr0Krl^f2MRo z4H?StC^lA0CL%%#N=lFB(bGc+vr}kPzWBHE2c}6HZ26-%d*efNDB>r&$@AiGk1bc- zMlF4o7{6Ab%^*rmci4F{QnX11Z3J&xt}3=BTW@!i#+{FOT$bQOkyC!T9p@fRvy!^9 zB?RoUXKt2k;oX25~U&R#sOMAM0YzElQu5_VnDch|(JrtSIr5FtB^1I8d(n zHdKmh0a#iiEbszA;>mWO;O=7oy_|L=&fgHgoSg9 zq-!F_tdhg1kU0P}ynJ@2Ekb%?n`YAHPtG`hR=tVRbn>*!Ttjt>iwg#cA*fKlNq49O zKJ&{&93sPshn6_Q6=&5W<=s++MwEhIIrc4YuNqpfE%YPR`R`#im4KMYd%73mu|OgQ z_P;|HLNx6_V7h#E>&2+B&r+Po+abZIA^kcsNje%z42`k+LJ|WU&VU=-%&BciZ~z^S zDmtlN!Kh6qrs)_>+{zL5n<(f$f*!Y_pg_a0wok$q?PF(ao>8!PqPXK%Vz>rQ7`J03 zi`T`Y?7^Hxv1IRDCLG+MoqcX`nP66{WHU zt#AS{bC5cx0Tnf+av2)EShq+K>TyK!UJ}9MM)?dD)o!4z&(S)?5(dN}>sFx@nDuJd zO^U7>VTcFmFi}7;{bCU2f6^($6d5@Z|C4pvn5hf*kf46rhyIV=F0L7DB-$Z)9~Zxy zhpEKJ{x-JSb#0@A*4FIrDj}eLIJ#gnwaRx1Bn^S`WXKbC_VZ&`J8`vwqjTGY@pjRu zH8Ue?TJ8U!+n8zldSF4fxh4ku%A{CgR>?e8Gq)5(U^OD(&unh#-+!j|FNiATS`^%n}hIHQ7)v5@&K+>SZsiZSj)&l=$^|)nI8Gw0K!Ec#=$eVGd z-^@7IgBc|O7ecw4z~1qmv9|fDL;Y9y0ZvL}45Noh`V?g9=w8#NfpZ!(s7~=@aoPlg zz4X7H-6VWvLd58No4^-8GK7?82%o+(K-922@6=eUfneE(U#K zqFG$prcw|lCRvkKt!RSnDIvmYXAZMPyH_45$~rV)#7 zUweap`(F~bvu;g?#V`oHjc;FS5?27i>Hmqne>cKUH8k^+Ir&1fsjM;VmrrnYHHuQ5 z*%wjeoc|#s z96V%LN8{3M;iITx+npTdK4K&6uoz6*)b1+n`g#7Spp;fD`wau;S}}3B3wtwnGML9m z!RR7PH}`9=^@rGBs6bv!thnq_@MZ}3{r*Cw&+$ndq3DrzlK5A9`}YwrIWWtcvlO@% z^Z9~P6)%!4do*S#qsNF1dR%kWWklS0W#XT@)yuwr8Seckj}j0UPvpYc!$?EwQoczu z_j#}&+g+u~mXE1d-RL5@3~FEgWAMIESVNpaOEd%UW<{+~i1t1`)zpaU7Ar7N84+3v z!&^#uB;ow2xAY0NMW)ev3K#mEbdHlD+-xMRL-eiS)S)~!RVL% zJbM1wsyIC~MI0YFU3ObIEEMIm8B(I5aHp`$?+Luo_(C8Xd%WDFz0JS7s4{IEGzys0=8?2ZWCk} zR)D%1X2k|lET5zHxmL(F-?qL%O=~Wr<^Tzn&kuXGjYUU?pbcnGeTy3^G!Z$FpEoKO zmmmjlTKCAh@F+< z30%+PiM@xt_g98;S37~8SE^(w-`!PRFA9LbHo#_$Zo^g#qOMUPY3a$XC)Yc z9^R|Jsg&L?x^}>fHbgE_#DT+Z{K|M5dr?%*%#4-i_3;|`ob#;d)nzY2h=6ly$d9)#Jorz02wY@>*cT#Tg;#>9l1Ny z`0RIzIU?~^!^+aLa^ZZW^4tM}D@ydKoHrayZ*i(~X_ZeHwYoL6$0m@)HsVzO*!=@cus-9D z#)7G$=Q)A^BGh@j>|_A11A79X!(9vL#;KS5cA5`Cq8r8qX-F!Ewxqs0D!ImIk8m}dpGYWFvfbA?L9tvp%h+XT za&s$G{@@v?1{i(MD2WD3!yyqP*Mh85}%>6#aeV z6u;#}@D%kXlRtE#sq@2Oh4DcDTT}qxV9t)f^WbWP9!ceQeTDPVBUThP(^jU_xX{%5 z9gY6udHHC0d;6+gTEta^+54q}&W4@ea|ZZ>x{gC10OmYjZTJdW_j-Tja@r&QRw*UO z>$95oa4C2k>(}CcS+sZ>157N|1$&amOLl)HkxOISY4YRFxh%r=jKV+lLCNfWEnj^{ z{apGEmFTOUi5zi7X$6n@klxzJk9Nyjv_S3WdwU|)6X^{fTREuDZ3Ptp+alZTE_i|% z+9y7i*ylHnqyH9J{^~{_1WI-L4Dr{9i`(ze10|wRzrElr4_;ia-5UX5PlTS{jr-~d zmMgt%owe+Wu%N_!>z}}Ly#KNPuhW{~yE@FwN^7#Ph1gw#+n8ZH)r0P;Yx5^8Jyz58 zM-nB@QBC%Q>!OWK$lE!Vke36jYen4nvn&U634>;&_ICy>faV#xHnqI?BUd703mH11 zfCPajN0hqreqsd2E@DjFX6~E1q~Fu?bwp@E+&r=S=kDh$WyU@(5(i=5?a z3ahoonFta?@(@XytbnC#7ne-WOePaa5TC|ioHQ8;YRk?rObX+A(i=hkm@u1dUvC2Z zxg%163s+e@OQ`*NQWy?XWkm_G1SynV8DLOV<&P%H@Nrz*XGB5vvjByu;6G(RaHsvf zD>ON|r}Jg2Zh+8__eNoEVGeZJ{}lrRL%$OCM~EFIIu?e&!-j}7#!9Ru>w5flvsL?U ziuPIS^S&e3ZRXPS^p;n`2rxJakX(XFlAW@YOPx;^9VS2*-EXPA}aKd>!zt>$l_DFoB1#$(#=`*b$BGIZ&TR8S51NC8n z94)sSgMYObHHA#&pTuSxk2;RQ!)(pEi9qud}mC_s?;d2FGpM`p~0+;IC@XUBWEOL8EsF!#6~(Mn}l{e64c3~ zcmhy<)oWe!jtHU{RpN=O5y;r3D$e)1AH#Mfa!$ZaV7gKA@9xLzqgmerVqafV^k1$b z=jS`qScVCF5Aec8d9C&)mzS&RvpaG)Q}j7-?i&HVH; z2(npAyLTU>+W;H0Ki8~CZ`;g;((D%Uef@T^A#Y;vsj=cWCvRp1I#V--?e`N{=wHUt zS*;HC)1*b3tWc8ck$=vrlYeA>Ah6szxgxqTcUno`K5rAx*e*(SX}Dwgo)951&ypG!@Ews~^2Ol$i5e15jvQCHPjwPN@mZ1TQb>>q40xOMnH*kryvvP_>q zi=V8;K*q4*)d%Xp~mezVw#TUE8#fJSKTl^O5*TAscQG|4)e-e zT{A@XczMUCD<(>1T5+$js;6ZUx$mjbMxCqy?q{0a#txyqAO8ty{)6!}J8;1|wDEb2 z9(#5Q`PdgLp@jYGNzq?90Eh$lel4#%uQn6#)Rob`BqS zyt1DcK+c}{Z%#+ErG7qFL*>=Yf2sv|(OBYkNIPhPyG$Dhs}P$fyRKvgwti?^otHD% zI@W9}_39(jqLa(g(@~9dha-WP#@xRnvoSUL0pOl)F1Zp6e?H>v;?cn|Dxe+(=)M5= zAMpJ0j!agM!sfQ#c~aG94x9o&bLxM?vK8NMyG-e$J8<=FIiv8YIdwDhucNp$N0%pO;7zU7v-3BXV>q92_|B)h1AGvTLo~ik{g4+hP4M0)8@KiD}mq zDv!7VHC2JvN;PBS>||vKA3e~^iH2Y4GjPu#`%ATIjz;)BZ`zRS zv2Zr(tpkYh7L-~7jWvL z6ENQB`?B?>Qf5+!!Nob{+<3eb&v^IuS8Yjzl=t-;LED<9S9@}SnTJA_{^OKiQ{kTN z;+t*Hr0rNidjJnlbm#}(1#C~*oOV#^bKRNqe)`;iZHo(LUdk0_K4Z&Ox>O$-&hvdX z9o5+A_-C&Z3YDKtoG2BNTLjvyo*0RXO-0aLlI2sL_S+UGie0)1!enOb(xn2^SLZo? zQD`J)i1<+=yMZ9z3DSPL5>Y|Le_gYJ+{_MJy_+QBxCkNVov)@37n|3ZwBIC+MpEQL zkqAxU46ua7c7bk|$Dskj_^R+#%LdQKV64sFDtw*-=EI=yL}#txBNZhjhg-Gl?GYyc{?; z2ZKZ)x>;j{23|&Hc&iyL=|^<#Q-u=hoFob?Xx@nE8I~UM5+3 zZfWW9VcU-xH#%A_I+pKla6iWn-~xGWk8!*L+#(cw7#1^vhOMqteQ!Mt4Nkwem(106 z8-U=o-u~Gv1s!ly+jyxMm3?{k(*P0VKtW1L=?p|My)wuD!se@zEKFhNz)i<(#m}ap zK*-HeceCLkNe&N<9aXm|xGiqe((Yb~_cb@cOk8YybT*U{m$DFdOT8RDq%SACS(?n9 zhg73=!lGxnaXwwX6w#K+mj$c@$=)Jz4M_%Ov$@C3Yk=*6^f#PX<)-Qt8h3?n$Islb zA0x6q9k46nU%pz%(f9A}va{Moiy>6=M{A2=`&D_kEhj}h6+5pGottYZso9T_I%D)W zokH=3t4QnQ#8zjDcnBmt^Da1^qm;6D=XTWe!`Mu`K;nMlit1MyvyS}mx64329kL1h ziMXhMIYa7XMJGeQf`M}EoKxi3V-IV~F_g@b24WH}0y^v$w zdFJqY@vZ0Dx$iw=o%l}2{aPC^NXm2GytZk-v%CMkiRAY;Z6JoAw8}Kla(^^!#8}UH z>I#R&l>cHQQ1*>9T_hCIgG+_L`+|dG)t%&iU;q7T-wo`8r4+d$op#L&;N6)JF-}B1 zu1bq*u|nT3)%87g3Ey8};JyS4Uv2=XE`Zm-I|W`iZ$h|3>UaXbcRp;vtCeH$sJx`$ zFePZM+}BO;Z{ra?VA(GH!SON|kHPNSK_KM%`S?0ECO5%*T_G&Av|-z_n}U_?=ldS; z@%?EZgf8uhxN)Z2=66?@^GkMei~^-}TViu?Y=-~EucD>0q3;5`&w=mArvgl;Dtj8n zo#5TpG3XXL$-O@sj@2&cFI&Pd&1PiD0J`Z|W5#(Vn;j$%N1Uc5t)#uIF@_?y^Dj~2 zm5v{QGeUHSwza?F`xI)orr~New7iZqG;$({HY8AZ7ihBvN!6~|3kO_(c~xao#D{@j zOx2wLn~80II^Htg^X&3{^P~t_PoMpx^nSO_wdjPe$j2MVf%M25KQw61i+Zmro7@`e zq0vS%bi51FGkZdE4s8YEmcayv}evyr|Yg%4DD{!QE1o(ob`nk~w?vr3eD6 zd~{ez;AvV_Yr!1A8vP)I!nPSNatr;lbh5G#j3rxV%}E_8NO<{hbXib3b#eDq zJ8Sc#H;`v3XPt#mfVks%e!P*9BCQZ|#~+7P^ne1p#*fi8gWbo~;LK#3q1ox~*V1ko zf{CwYN}uuTW)>f;Bwh^_9|XjqjG+?Borm>u{h`3ShX(*v<1~A^IgH9M1}6KWBn*Mk zI-SQ)@K_rpu9*geJyp6qIi6QTcXzItP8)wAGku>=IUXr;n4Iu|Y2JZ`QH7^x zDx?7s(5tSzd<e|aUpo8kcdgA5u5d3=9RIXOtoz}GGFc5{qqG-PAv7aR-CZ^49 zcjtI?@pzYE_jVa;R?9)A2zioBMOD=3&8dBENL5n&UQuxq^Yk(TaIGHAa*PC|omd)< zy0Zixhh`M`8Z54h%yM3z0dlp&cv9*26guq?f6{Le&$e*s?cQs41iEVcQ5|S?f0iKb z4(mp0If65`S`CK6c2lFjyw;~_&eqim6ZNg5TQ}CtwE3Hwl>1(85EqK?oukDsH1d;_ zm$#`Fk6?2+^cPq-MM{tOjBJ1ztsj21E~BxR)7>caMbyVt6zq3%k?cq17Z_a!%(TU_ zH(&2Z6EKlB6;+)^5+YuXAk|p33+ymUS@@CzBoo)7Q@isw`=ZjXWH_68tjK8eI{co! z7pv)LHu~N!l)bUYP!3&>`Mxe-VRXdXTRuPU6Q2C3^yI>Hrb=0&)cJW=-nRu+Ll2X7 zKPi7KWH`^(revA|AawEKHl48XX*`;T!7QD%wzH~T)PK9azol5&aa?LXGCz6AbeWsDHuCAZF&$Lwz-pdP zK-quzYLoJ1#BlzZyXI6jhbU^2yHqDnQJLfCl5tOVa-?1QTMtL50-kBp=xk8kkuIu8 z(69YlUuC$9MTfrF*InY^=^u{{-eiS^X;AL!8-_1Z9jaaX9ZrYy*}n306UGF56!gs`Z-?&GfuZV|U{o z52sDwqqE%XF-uEJ`={gT&b=HRT1>F#apgZ$8==n*Km>U`Ph*K9;9EXtSgQ5;O7Hq? z)6@ZEgfyKmS6%w=t(DPQh6c+whm=r2Gly;KBW4C*jzaN*hFtxhApyzW-;kMr8p3xuu zH5NiB(O)90Src0;-BZuQcALTWePfSC-)}44uE~x+xWwzb^EH0muMaO?TY*ik>mEHk zeqJwL)*fAc^lgk4eP&B&S(V2xryV)#PR}qa>`LImIja4!>Me}O?n0U*%Tf*YA7?F& z$y)hyK;xv{gZX(|#qVa7!l|qZF4XzrJfvTr6N8qvLsmVO01pSfg}alsugx#*2xD?1 zFDo<*`q+=C6G}f}uRoDyuj-6NFUa*P z->gPvWRCmybw0l>27+`Q#=h7A-BHi)9xmEg9gzg)@@^ku5_`?Zm(3JRDv^7B>G7pMHrNH@)Fut5zS>*4CDowS4Z(iL#M%le`33 zmC3REa#fjyw0bcf8CzXHoj)e_CZ6GRaDhI$r@C2rFXq@Sm7u=LKVseJb9-kSZ}r_l z;wj8FLK;Y3LnsK|-x@RC+_oF+Uar&FC=Nb-c5-A*)nyCNM-aHO)6lQ;v9j^Y5vx{z z-93!-d)!Ef13~TXSX6Zw!U5rmr>}^mZEIpsm`TEjUxzt&B|WY}!+rhtIv0ixySIuF z7|(Q)z)p-849}qritvZPpD-yefD5-zSAO(+=M~)jetC6AtNj%532bRa1-JLX#)7_| z%Tym$G%g!814DH~gUf2O0{|?0AOZW_>rzq(0K@%Dp03P;jW#P6BF&Pd)HGkWnl9A! zefmg~q)kHjMapOP_V$P^scO8*{Vo&wWKuFm-X$d@2EaLwUlV;d7LW#auYf<21tO7w z8pJs1B|FL3+}A$%jj03RJEI6Uu6jalWw!>W@@0)y=51DS^e*Oxq$Q=UtpTsR+;H8*xWe+ znF1QuQ-fz@KAPfqLa}YZw`pr*v}M-wIGD^+z4SoEUzB)gcxa{W#}AQLi)NuZn44>0fRB!%%fwxC;xw3vHVW?!)#@AuV=5T6O zCMvf&%VaJ?2ieCc%k%LVo2#nRxq6bNHFb6OlZFZ(f;nqv%d6UrUdHin&5o$ZUe3HuxUu_vS0btr zM~t_$Z8JmO@z|*Sz@3Vz3834QL%{KiL_+)hVzCLc?&Gtgp7+`Ga71P#1K#Vf)ryji zej@re;T%e2_iJw%XhbO^yU zuL6+kLS5%_c&E87m1~#H>ZuZCibwO+KABsw`cVaz^_Vd?C=Tp*8RN?79T9|{{Y?{#W>^jhNq7kns;dd>-2>zSkB(Wji*jeQ?y~F$>oD}f$$e87JSf)C&*=rwX zPYNcCLNzgRV62$@i}yw9d8{o@plI*(is@*(`6?HO{7TzVbIZcdNmR3)LmGalON8`Wkp9R+&Z#>dCi^?Y%@ z#C$Ja#2uVq*x1KHSA;(|s#ewq4Zv0Q`_AjgwC-JM(w*7hWX9YqS#sh?`PfA;ut&Z! zcJ#N363?3{jWF29#O?JHw2hQ$yA>r$*&l}@NX8!HG`t~L=~tES=8&ed>{ej1xK1{* z=9s)$TUaDX7T{LK(8MKYq{b(E8#Nxy!7XbN|^#>DVrYy=lWVpW{p=F zO0-39)HH*wfDh+hp_+1r>8^2!Lu(%AS20o0(A_EKFy(LMVR8hdALH(We`()ULNG@D zRAkuw8g8Gp)S`8o<JjUgC#Pz%as zc~&_IR$0ABq~p)JZR3KiqvlPzd|gLn0_Mk24bQf{BO4bWR5`%-+gP28E-&TbYC2ds z;S;Yy?(yC?R**QmU-PVYBW{8fQLiXtQ5p2OFba(9xaW^SFPMKXz0#<*7;bkgjKKA? z{ldZ<9FRs`pg4SIQ^{A8mA;aimTw3?k_YM1h5IiVf=GBRV;XQgOF}O*b)vx^ep+aY(m%1mN&_to#g0vks92z+aN4;;p3W?1O%oGNheB^C@63L#BY#NSl5 z?lZsDQ=kS*bK9b_9n@=C>z*dLP1-uuv*KLAyYv;}ESh(pz-uWX$I~k@(WVYr#NO$}Eb>@2Xq{FuGS`KBCfV<= zMf^W4pnDyq>Tk{F55Ot**~_cvNhQd?kb7yPX>qc9RKiw3ohUWi_Q?C8@BAqWv$|U? zqhsNK=~ym52@Pt~o--P=e9h@HZVJO=J5-57Ql#8)yK{z;QnTvyGcy}C`lgHjMIn_& z;_aQsV3|1GkG^T+Iv9-^({Y{ol-Wh=3(w;+nt@2MBu<~EBMrAy?F}|uHIcoY*2;^P zdJE-Bocb#~Rv}w|(j0hP+RogT?5qYwtu*b9KxWKTwc3n4mu64x!gR~5D7+@yA$y#p zWJ{cF6T|B|6_#qlSFCGGHBk`jg^^V2U@JSnq8HR41sW-vYyl2JJ71vB3x5Cwig z{e$n_>EooWX9(!}-irP_dseIFUGBL?)25B-MdlnRxkZeR`< zHsYAc2yJOMhjUr|pXABF|TVKJk0vFTk;`kDh?x+ z!Q8?Aq~!P_%4aWqFC;J-NAhmJ=wg(9z?qJft4w>j5OB?IIn%+l8=lsdSLKk$OINiR z9z{$b$^KGSW=psVp3Oe0@v0JKHFEAsUXbex=i@nQ$D?j*%w}4*P%7PZ!c-yPo1L99 zY=4ke%gHRU5K$`3&oA^|gpH=ppAcM1X09vQH_i{geLZl-dHh87b-@>ZzPKA&c4g(Hfw2WEw#yWF4EqD@z^xO|b2ztoze2rb$F)mTya9w>rfvZM~ zp=W6JT|&=X^eq`5$-O$j-)C#QS6N_fxW~>aaM}vnlG&>*@8AAT1OXd*FR&V`|B|_& z-$70x}gNu=H&D!UZ&>I3YCv? z-S?|JJ4FgNE4pmmHkj6@$w@5QH0dPr&g#H)S#1 zYA0Df&VIrGLg71S?1_$_r#4nri*wfLN3ApxSSm488un8MfT)f1Rh#n~uk6-_orm4> z8uX1tz3zou+A8fo8lLt)8lLI@(eSY6m)5-qUd1r$Y|e|c(BYNA+Z)9~Gf>{UD+>f| z2Tv9nb2`pFvvWCw*eoptnFKc3w|F*~$tG%1zmrD0bj`kG^$SvD)|Ob_6rhS$6YcFP zF16l0+=$IO?FKI>JfGa2vZSamvEQEttg#3}1i2`f?wSJ@SVxUx#kd&=sNv?*1qs12 zAuM*Pga)Z0ZFl{p!Jk@nc_tK(rEjz)Vn(%HH#PPuXNucHwbgZ5q=fH%yjj` z9Sih8rtf*$dV=R-V>2K^RGkhhnwr9b59r`sra9N=zZwp=eU@o&6aHE0G_)G6MM&%* z;!DWkjnbH?>r|(!lqo7riG3N+wnA!NK$_hx!NFg_GS>NGEvV*G2 zRaITkd(H?Y6HETN!sq9XERr=rSZn~KsEUsT<`nRxB-#6b~ zbK`2YPQZRXJJSH)4dcM~`5)hHTSDvZtU^Mmu&{L9vLS|m<52O|wor$ypuMcTt^?GR z`kMnEpWV^gOWVm!T|y%V7yIHjY_@%dWOag>W$9sr1~k1cW6HalOmP6vOdBbiWi(up z5V_}J*U{zxE=BuC`4lD0Qge7L+y>}vetjIFu>|mQLpBXu{7?0`ZQ6h8R6)C^2az4j z=RVCyWFM)}`?~ca#J5KZU@+}uMf+5q(u@3SddD$H4!E{o&C)iFDXQHvds`=w0H}=7^zGWw& zRt5+`7~Yx^^{wxeT^)N;<|LFv@dPH_uOyvzOMaqkM-q76F_YI7&ont?8aNu*d&^Hv%w*kv9%@kp*ahLkjQ^>%ST`V(*2VR_X; zav3=ypu1+j_l-xcz*=^bi$iYP&$2g)=pmR=LXr)fi<&96AZJQ`!aRtsd{)oX$pwXo z!T+C%?PFnMlPq0Gqh5J*cs6gQ*foV1x}6(Ju$2mc-m8H!^;x&DB~GGU=VkVduC6Yh z?W+69iw4NB2gn4^I*-jLz3)!edS3jo-e0HRUY~D0eGHX|-fTQPJaWCgk5k6NAEtA#UrB;M)c8#lIMNDz@so)$HhmI{=7-ZrSFG`?h5Xi{_9qk*ebCubl~A_YNN`z4eaLERrQxCp0+Xa zmBIyC70Sgz16S)xpVK}$%4JUcn9~e3xr_#b%a&R3q>$`!pEeE_Uz9>ET@7C6ffP0% zy~Ey+fXwUa+?Gq@X4?iCGNxD1u;V|P(^arQ^aeG?bT&4wujwHr3<4Juf4I8B+^LxYX(|2h`8E*5_Mq(!(9q^G{Z^p1UH?x-Kq{ ziWYLTybfcF&30=)_>cUs8&01<3c3{wz035u9Rk3udF)w_&7ld36G07>*FqaAbRqRmyve0lRp#a_zH3!J-Vrc}WDJ~*uH z+hTt*i^|jyCM;qzK;x$y9oj}YhI0@?9MqxN(Pv#)LUUe=b6p*}cmjZqob;#4$m7Jd)*X1ov z0s~V=844KGY^)*2Z@N*WT*CLmKtVg7 z_B{ySN1x+c44z9GitkyaN`s^ccLYBq6a;8{zKBK=g7@fBp6}S!J@+NjG>0=~lV!;* zEKg4ntYzC@j+#Dnm^ASSJU#q1%OM<{m^r$=ns0n_omfNc7PUWOlM>NT^Hb~CyFLOS zY+A+URmbdiUO4*&6NldBwqW(fbIiI22?0aBE|?al{(@>sB} zeci7vFK`4V#V`rBiVo+g0$`8SKDU-j{}ih)9=Z9;z^`gy%+-P$t+l?i1YQO(aqcf= z(Z>o~scO}|YdLCk=SNHpO?R5Fe9s2H&V9#}tiGPdZLpH8bj95I?_+QnwyPh$%wuHz zM`4Sp%pv$&ttdZdUw>O0ssH)rOuyO_4Kh~<@Dc~+ApfFOsvWl($rk)0kT6PK{zo9W zSrXz9{jX^DzX}|HD;It#R?Ekb5(V;}PBNJhY0zT`Zk_MVl!Tw>$&?^P(vY#*u|ah4 zgBhjDaO1ohDFoosOidHIWIew<5bNd!?^$UNp;V{RY$5TvW}$bJro8_sy1(mJZJs2r zrADPPzJNLG{dY04U7rr!knFCtnhix1>|EBtMWP32sr;ehe=gr?1W=@0KRy*KZbOna z!T9W|HM{Qodi#_6dlc$J1Gc)fH>q z8i(TUuEpKGI23m%uEpKmrBIyW?(XjH?(Po79q#J+&bjL^?EPdV*-7Te9PiXBiAY_C zjaNXxAZbTqAsaHy*WPlyBgRJ4fE_ZuD^flyQ#~yV(;~tn=Uq+ZzlCxmC;c{xyB$^i zADIGV-}N;f$l@^QE4ysolM}r#`ssO}s2-4R@OKma&`h))MP~kIp)d%tAD8w z^War3L9}FL)AjjJbkJ>K6nGTQ;P~%V1IK-Ow(r@z zWc>sZ=nYA>efg^yxKhC&5LAT&tty?)Uh$D*#{+|6Ct;1pLD`eUfn?EdaNZiPBA z9RJIoq)fj^%t%qSGVlRJyIk@F4VA8u(}YME36F7)KF+WK%rQKOFf7VCkxcA?OpvcF z4e;VN!5E25Pum!2E7VB?vX&Ih4|Zp3^fd{$DlQ!)!q2$n3G0K>mSQ9iB>-4Vtrs zP50`r%oZqRDz`9=F)X(g5bsPC$)F2A{YR(D7ksGQ%%Bj+$AV-hvAuVSXJWIH&ya{Y?2 z%Vu93Cc3-X35<^rCoR&&XwVetV zg5((eU8BE~);@yCxvMcv|4&1uB0`H7p@?;jSAqy93vXrVrq2JGIUp!@T%JCB+B9gy zu8FS&pu<%xB6iW}dOShx{ecvD6mQBlQPeCv@@oJi$J%2xq>^1V#Sm*!jm!H#Eub)J zUAn-#gE?`n`6Nx!C*lf;<$kT;+l&@-6(_)N>wL1PNVs@G^WxXA*~@DPns!p+!w0ku zE)*u-?ss%4k^B3V+0rZUCVV29EZJ0!3IM7XH(P=9U)vxUX-uen+CX>|ijveXQcFl< zw-^H>`>YAmz=IHfrt0PNcT=yxFvZfqtO*ZU4(-$mp^Q=uy2%d#&vEoNzwU>-P2KY4 zJ-VjryVH=zK;(@Iyw~88hTI?O&Dxv~D;JIZKPyxm%gB{7FKP&)J>t6y`yAG`EM@Z7 zDI+9l_}h^*Y97w&WM=cO&g6x*sLZPoYXJJ3wug0ef{pio6Z*HKK!A{sBds4bMKVE2 z4V3ZkLO8{2qc~awwt?o9QaV>?4#X#ktPapnRopI$r}S^HzY7MT((NJl}rNxiWiylCse06B4tr@cl+K^R- z2@R}Y$-u!WVQ7jn%~Sm2kT$%D4B(u2OJ-=#5lNapveOS?Q)=sGtWf06in6sSHfLqV zDwNUmDlm>*`haCxydtq9eEcBd1m>yCQk(SX__htL8*03s}s$nloQoh+)4CWeUD zStD2cMu?*{$}-yodBj3dS_H8T66@Xlb@x(kY5OLnXs;a zG^xV*Tn*B1`3hfoR_Q`6nI0W0weS_9w+*tUh5@J+;ZR#NRy{NJ#w5+ z_nni$1=Z-J)C#k|LrHJ0nhk$!=| z8X#j2D4_#=_0jY&&-{NL`5%&)bzWw)K9_cf7}V*AWg94q^;kQ)xuU;f*%xv8HXBje zshKHrn7_tOuUa!}5>RMLhw`uEi8d>6hKZ^z*_^WWbQYFK$&?KkyP#uH6GYF)Y1f>8 z&!}P4bTz(T_{PlqqMsbNa6V40whS=7at9UzxhekRY+U5Yn|{1|u-MP%JR6uIlN_lS zNYTlcWUd$(r7vqxn!CmPBFc31<>tx`r+Jn~YXY+_&+qaTzl2oG@Gz%vbAAabn85j?(A%}bE8P8<-}A}l&@)RMntWova-$34559Y2c= z?-E~9RjiORGkosSHkUH)AVd2AELBFzSrbXp4tTU@Z6!;KZnK#>#VY2_HU#z+L{vj^ z`_*W(8yl?}h9|Dui>~B4XQ4bz0@Pv81ApKf#pC&hL~Qno8gdWLwvi+zydM?|bwlZ= zn+jYR@#_)8ssl9K?r^LcC_RO9lnP?G&8bP_>#oR}LsJDjz0rJ%{h8Og{~&OrjZ`wK z(454|*EV&ElGfa>@O~wTW2ne-3yl~sZqz8Q!2N4~)JSB7IcK8&3D9mB$;f~U)IdGd zvKIi%P&P5U?zDIYaM#2seO_KeT^{{3 zQXh+=h6YWd^c2TDx0^WP8|xj8ilnS7*8Od5tz&KNtyU_Y2Lk{4Z)!R^W}GNlS+jR~ z3wh(+seR15`-l2^HXeSRSPyT^%jp*n1edm`m)HT5CYSC99P-e2QZUTj9+-`8y? zM<%Am$9s|&XEvdf|5=3mh{>z$36bV;r?4?B!lmiez7{CqkieESuN2Eg+9Eql7BeShIABx5BTFga+f^fYkKxId z2Wr=}sU_ij=Fp%~nILUU`lZ3Z@y47UwMpP-6VHTr+lo2MBb6f;L9Db|(6a8Y_$2Lr zbiD8b8jGx-HEb;&+R~OY4rfewQHP-QYSL2TK|wmu;JzSb(RI$^821Vy_ufh=lK4Nj zl98*)W(*uxk3(^dx@P{&uQFb4cUIL+epQw!PFDIy7#%4Tk<<#+kd5B-$zz|;61+yl zV61)&kXm*ZfDiuWJc7Zg0*_as@bx-txd*_GOmWf*YHE$l_jbY9)p41Wg@K_6m&}d-pi0}d+-iT{>zFj9T@4@tNkSU%y*7c!h;Z6c6AGT7|9 z9Vv3(!`+<#_@xa_(7JeJGaLGoO29=as`yg`(!URe)kD6L%uZm$;)WGCU`RtqCt4hy z4{+5u&uLM93*Je;WuT^E;MH_=J9>!v9xX}N`Ei(qK*U|SH0`Lr72S9^Sf1W~Gm1i! zSa`TBTsU6L91jZ{DTX={6h3!0nQmoK4$S$aQRBW+@eFR&KCxCHcjfxV(x-gU91r&t zQTU z=UAjnXxKoUkpyBGg;Ir@9=BD3Tr#uu;afbr#OYLe?^doFz<$bTc3cx5*GatbS^+7+oP1@ zRraiHDi2H_kDT_`PQl~*6g0{yF7H$ab@TdG+R6@~azusR?L_MW?VsQJ5(5Lt zI)Y}K+-n?F&ODl9RzAYlR7CBA{lSv?wQD{vWx2VvD0x(b{Q}s8{2Lq1*tt3aO&|N} zK*`}JE8DJz_3+fbFx}s0nXg2#V~NXG0XlSC&}|kL@=Jh$XkwYpEtwjRAy#nD{B$CA$78{B0DV6Qu$jPobk!2)mDSiT)@B) z5{}l0Yw`R-=5{(XX5Gm?0Q}icS!ww$CneP)N&}EqGyuZaC0U!R6)7@FlB9}-vtE_| z2$1i#qqb7JRz4K1ViYQEpjfiRxK(S<9&g8k&pw1`7z+DAyeBJshe%6b5j8L%;&2O# z6FQoD)%BOBFXxnhIVVnCA!vi+DYE&EV2u;vrto`O3y5!iUEce|r=hJhRx)5f`pe8% z6*JbuK>GF#=m(+Ic*4Prgl6gLJ_&HUT##N;YEiO9LDXN8erbOgdlI&+{{8bsy(A=L!Z7%01{9%4r#F?2Dc!RXV&J9`E^! zn7?n?=9fq4wX4T*t3{7q2sk;0IjQb_Nk?)!v?jI&6z%Q7XQtywx1sGB) z)1lsCja%}ZZKbjJ&eZ<;MVIdOSb^oi*JU|%X5X+oj(g&75*}M3?Zs9Tcoz#x?O2G~FC0#jr(yf}!bwV)-MZgf|Cw(* zpq1!3;Nr_YD34JG{s~Z8TqOOut37;)JpEIH2!#`0V@MNWGB<_6FpiIOVSZ36=zzqm zhsFGzwqX5R)6@h#Q-eDpYi6I@-2Cf~gbs4v?Ezi@6CJoSaA1fh7QMxy*LIo|)5;(m zh-(iZJ(soTI?wWbQ=6y>eE05W{c5fB(Z( zv7|3l({IJLlLzqJ7>zlb)Jx}YlYbyja>v9Yj;u8M9Mt43poZU-o?R)FuyaXJ#&11t zfBvJ10h4hTHgdqw%PMcr0eBTZC*lPn!?=gtZb??IU6XexPpwiR2yakoBsLFN)Fn*F z6&&2f$}^lwJX3wqraxqecz@;;7Q`n)Y# zPH9&p8HGMhrPn>(l=<@g96dig3|VgvK*<~2lUZxACIbUran=u=;1`vzBrGyYbukt) zyg4&5-8U?34POfGurnK4A@clhXalwzEnH|-a57g1cW)K%1E6@z;6ku5K;R%sIx%AM z$bNEiV)2ttTiq4aQ`${?G(M~S(_?COaeLt$BR9>fT{8) z5Xe40WPP;OTHNJ2GW=-r7--V-o}ZcFaP4t_+P{0gh>CLF@_t6C{QjT0$OFtpO#O?f z1h)nJdIblt3#fivrgUzaGYHLwlpFHNbG;A z>hLHy5sQo?8bItQlJhooo7WQqaOsYn`n&|vl#4?BTLd&1anLsq1V|X@YI_Qb8fy;Z z2;tcBh2($`7WOkH|Iq?z`G+`6Zy%P; zZ8au<8Nwc(NP16_g0x($6W|Dg#08Q&{$p4CRV>+uk+dXbo)jC(PWCbt_W#&L*Ia-b z@81Dwpis#+0SMV3d1-!rgD`k3`E(>?=UHi1q&QJqcr)DFn~)&HsQ)pTp@IX|PC-QC z7y)B}SAl?i36(j#FxUiGKDT~38Y#0ahWP`Ne@xDZLBZ}DcS5%P+~2&4Ox9=8SI)3cu4r%9MmYnbe@#I40BE4Z!tAmHk5IXkTprz@I1vyUY79R$bRI>kob4hoqo{7bLusa(C~gq6 zpGdZzli~l{WDq{kR|f@sovH*d8eU5BpACa|W`PJ_SG!r5 zuE|0c-&M4~!s<)OHo-or7=F_?@Wkau=SSL7LuIs2)@|NTz9c5MrVKwS8WSLk6G<2cqts`mg?` zP-vSWCv3oJyTaa2*{colY44;@gelO;;mMQ#4u2rkKCbE|X)`5#%0`V+LhWRK+n^|847Ph&}P_mpZO2mY>idg6hB;qv_C4EtBoQWScI&e z$bZs$LDgcyzc8bG3)B4a?|V+yeeA>IB(E^Kp*G_PUj#rTvHt^}00B@~-C%kpB37qR z7JM8+9U;W0EN=4ARPf_0`UU)V#`-#lQ!=1^(2RUm3m-&a^?O0pwId$ zDtrM!R{vig8~brmocI@X#DNq`aw=e*$o;T#ff-jq*UyNgtl8)x#El@;6bV$4IO_k9 zo>dim{o(2t0aAegVh{N*H3fJ?g?8!FNKs}_^Km;w8PH_NNo$sy!h|NO5UGAP{e$8G zY`LbTODD?hXXvB{R0hRTfKqath*}{{lN^?B7^{)MRuKqL@xV3D5d&vzm{u~3-8f++ z4V1`GQFx+6ixP+P4NDRvNm~XYHcTD0tmH(hCZ?0I2MA5x7QN{h;hH+7sd^Yo(oYw~ z3`rUE@E|3j7b-plJ&!<-#O*oWG@Uc#T75bXd0!_x@bEZ)loY1O9g{%7d?6u0fB^*~ zvLzPcp$LwdCb=^;V$$E+^lX|^_v%o0{@JB@A-!6u^OK??|M@d|=h|W{!R(ZEVDAP2 z$&CP5B-7Ge~v^*xW^_}0a_#cNZvUO1LOY~d?< zEDwU3G3%k0lm8n@s_;A*@4hjVzZx-D@ig!G1+JYZB+pOD=UJ!>Oj;VI{beT~LB-0V zfK4#9Ikw`apxp;Vx3<FVs}#cfgU4-TSeMI(3kFEFqVGU! z5(mP9`EP^bs33>UlPW@CsKV=*YC9Jkm}X5~+&@FuO8W~hR%3y)&cItwO!f#Pg&_?` z=PKr!k*Dxj@k>WER7H2K6JZu3+mO|PjP zb=$SWx;UfsJxL|0>S@^Ayv(avh#`IV&RCpY*|me0w92IB3kVrxc~)f8tu1feb#Pq2 zIiFbfJl^yvr;oLbgL7PTRJh66f?i7?Mo4J4rI z)$=Bz+y@5VSn}2`ZVKd2l@~nZ{J-e`vDA=0rTvkggUYfg)Im?jI-qPDALG%#3e&X( zb&shpcaWL~ygY1~yFqMC{rvvTuvJ|NZ$Bu-&A~yMJwDgDdAT^Ub`3R^LP);iSm<|e zUl;!`WT%~f3O##4r9!>K+!27`g%PFkgQtNJjX=>0jL_1$^#n*=vSA0WCCbIRz&Svv8c>UmR zyH*;RqAo`*o;1x?n5;WIy+zAul9J?+wsnN*aOPDmXq1&AXgWBS5TbVwTQF7j4g-cU zTMW2No&8<@Lu3-hu$x}zOdxV%pX#*pSXqz)o(9+N-)|il-@w)o&sw_GS%2ko{QznN%^ zuSO^VmEWX*$N_Zbl-e?OMY5GbEDO+4>hJ1 zgHnRjXFl<}ay}y21X38;nz9aqQr8wO!`!$eF}}*nI&%IudDryuy}Q!olfCA)|16^* zQNzlzG4!bbU&WaK%YEG(|9z}5r2oHGiozr!?wUJy8_u}5k56bU2&|(=9r5XA4U{gN z?DJ!XatV3sekm3QcgCe#%QrEL`KoxY&oEBRl+QyyJLLpYm`4G3YW;Er4c2GgFwC$V zm|x^AR@V1@MLJQd6e?uKYT>7IO+JBEbuOm98r7YrE~_kR-oeNJmT;pL4I$3YP8Y&vP_o&7s4cx>GA|pJ#o)`v-ARjO8jD zK1-#d&uemI2!e|zNa{D(G!E!mP-op0cNWFq!)cZunmb4;b5?Tk>n!*GZy1Dvd(&!C-s1~+#T2?m9BSF8zGNNR z#*1hmO`M_$-+=~v95i6Tn~|jv$YP{|bDo^@U&rVFE*pb`>@j9Wt z%3)x5eYEs#oiE;Kd2W`+?#;Yyoj>Xx%34(w^wC4Fs-w&jd$|7!`jD=Nno*hx16K@% zxPWx9e2FV#S8I$Y(<{yWhlzo3G$2Bpux$HRH-XHuEc<(jREYSJv_&ArG;8p2|Ohrby0cdoV;< z*gD&As9#}TKp7>J$mcHn6i+x%`8COk?VM2&*dVdmAq-i4_y;c1z4YT@(dRV=ZkW7@ zxXi<=hFM0>wY*N&r!~HNp3`K6Cs~)rIF~kNI-=uiK@X{@zrR)VpdbB^3Nuev^Y|gQ zLl)oS%!UB^i&Wl#coa3hqD9^Eckqtr!@Z8N`LplUca9Qh$8o4>3l_V&Kn|iY!Ku;YY_j^kA;|5Emvh4gG0iw;-;@0SnRK~x zQLBYzo3~=ayPd}4K-ry))SY4?N$>%;X=LnUQS_m}64Mmi*bt@#6X|EBzzm8QA*=C; zgvx#|{ORqpR<0^Z`d5+vKTXjO&mAWjXz6V&Y_ytODyI1X=)w5$X)5$(*XRBS8Zi#G ztavi(x2jovrYA5E(bYH>qd{zZ!FT|T4p(Z^bw+o2UzDX$=jcg>UQ0h%^x>eXh+?#V z;AAu5frU4yu$o{kyW_X4+^<(ti`hW^r?|I6Fn8?OFmViPy-?SeYC&*oRRQdP1 z&t*!y_-xf35|6HCgXKh{-N0S1ce47g;k>Q64Z*i5q$Zn{Ef`~#2Gjm38{H%`*3e3P z%j35LJwfoBogl2(IQ`VGbJxK|j8$LFN=>f*3hJ%2ns{o^N5axQZZ_Dh?z7xv=~i6@ zP9b(aOxhsRZ*^Ud9~^!6>m6yW%z8gmCMMF{Q8jnc?x0N4aJX)!rr7!%VA0lXW-8&` ze7^Hm$sPZr=iH|Q*DU|OH&XRr9j|Uj-{rDu!a0uk?s)YO^7&?vF+`x+$JJSo)N6_n zGm?a%*k?-TQO0>}(`xyl3Aowy-o}i|kS=XPdizfbTsK@BDbkpKblRHj(N0-oA3F!Y z_#k-J>jz!5t{q0Fs8}H^HdcK)&b6va2&s@4Hbxc6ibaL=78~7%q?t1QQ0eRNnY*uA zzL<}BDM*&pY_+mlDE5j9!Tt_pdHP-*a{74{PSJIJ_BhIl1BJq596yb^bQV^aA@&1T zt(0rj`5VTyPGZxs_Nb^C!++su035% zn?A4Qb(mnHxN)!-Y1?2fQIrgw3wZVU>?YY;ik`WFe0c80CLdLVe4LDew6k;eKeUKo z84KUcR9eXVa3y!b{P*f-&fkB}i3RsiD5Pni0y`v47?-!e7 z9wfUm;;2fMPQ#_`pe8IwC z?h+!&tz0acs(KkQ8k4ewEpPK|0}l?Ddp3v?8>0#;r`{C0)gMeR)xHd`mBKGFg-1Jh z1RcA}O+tfAHOD|W{CLf3j;Svca<;HNafLK3u?#G=9Al1OOsI_RDSRQk*}YpybbUX? z|3V@06x|+Bo^o4SUf%*&6NB7AFBX?0P_n*vCpWBIJKw1#Rq$QoKj0$Mr$4ly&g z51*SQB{;VLut?zJX>T@*XT$B~AX%@nvhw46&fJs>w#IEFZtLS=CWc_Z+>jA&;p~1& z;Bh^HS~=&}o|~fL%&f`SA*06MzkNvFoffLcjkHzmHvr*{h#1r3b%*Tl$5E#5$Kusy zt?ib8pCp=6zDZsTr)S&6)Q1VdyR-;@Lok?Cpv}|6z8MiwIj>l`$>b=@`s0}+l`Q?l z2y+&9(aqR`g#Y)f_X8wL=$AgUFKq``j}#}Y!oLdXn#TMkHgn*O1Wa7o(z!BQ=HyrM z%LyR+2Xj^*K7DG9UUw*q;ieny=Yq|CI6tT?2lCi%HuDc@k9c-2yPTVQA6*YJgpO$^ zm_2V&)L~7Oa~wO}7O5;Y4KKeRh8(aqXd$o_@V%e5f;U+EF^wBfq$(gH!{~~tt9jKt zEYIv^3|Y3}%h@XWA9!+(>#}vNW-m=->1b|A79aGsv7+6wM2sc0;4f?i?RVp=d(h#%z@BA!gM(4N z_R@O6_<5lHg3;1OMc?K|>Xacj#ro*IxW@W+``K1(sPZxXoM+{}7Nibe3_)S`W|Y#d z6&zGEOfu@AzZfu+H;C$8FT0iQwnvn~4GHIGn`(Kt|8aY{U+_FX5zS9|Utgd#QQR(lyTZ*>kSaeW4azjS-0ntis~yf-ztz60-k6cuWl##-bscIe?J z$(7aBxVMIc6!a1PvwPw(3l4!Q+CaUpMs9H;*~d4NopZNcNce}#)l5cIL+I)uW7Z#{ z3Rn}E|2uWhfJb3c#w(V^doL~$eg1`4Kn(2pyCy-TQ?twRwW+Z&DJkh@^Wiq@6HH6O z!_(vQu1K&_^VY;i_yeF%E(1#y(c8W4Vuj9`9dg^5T2yK4r@x*ED6VJb``VTjH^4OJCzUk3dXy;Mfv^cO z(vE&!Qv|&0n;S;ZHU2X{bh8OeR&j1&;bhpfDyEM3>wS+-JTfQ1B~*VCsj}{@RSERj z`~EFy884Cx7$;Yz|4NtXZRRNX@bZ>r!v&8(qrgB5Py6|~81!nJ(b0G1>j;9Y0>a!+ zmxsn|Y3Qy>&D{F_a?~bi2}|6FSm)b9&z+`As)U;Q^7)&ie0qid?V|`fTNn}SswHPSm_zgBC=P~6i{}I9@A6^#@g|T3VqMZ< ze6D^&4<1FKJ}FHK#XnvzgAB8&U|@~$vX;?xmv_w_LlC7t)POfGi!5(14@#>4o)f}XDzb}<1(g^4*4Q^2ytHXK71{Kn;Ek9yXYWTu$^o<5^BQY&~{ z>bBl-*2aZM8^;nOPnHHYXYM@B`&%ZO#46GA^{c`k?k<`Gc$#YoevZUrZ?KCtXvBJ$ z_m}y2y|~_73U*=|HZ7|e!(3@=YaESN_8Gadii;p9jIVV3o_s9D?+wydb==z8cgYs1F& zr?vZe%@*JgpdzCc-+bH9{`lyH`nS8HvTnKu5FLYBE+Ri?=fX za9iORTA2hHkzPL6oX@spK)%r%r&9-umi1A}(@#vY)HfosaXvnaBK0&LD9E&}uD$qs zWkpqv5b@?TmpL|<)GVFcfTc)2!-xEw)T;H$?s8pvxX&P<_vla@;;cs)%VOT6TFT-P z)=~zO*Fp9wRjuMY-q#LLPD^0RC7+*b+_q1y*Z^xjnWL7is%cAi39w|0O_GJ(k%m~81t43ruoWk1pABb4z<#ZhkQoBuwpy5#{&_E% z(zc))HQsR(M=e+M8vJSHeiI8Pe2TnA+$?=To5=p0U+wAWEhQ#5GTiRJxl`CG&=lDr z+%6s(m8NAe$>P+ASkM0W?0plUpd`1Fh$rU)+8d)Ra8^FEpNPVuDhc_OBWzrw)IX+g z`1ITHdVymN7qTtX<>KQuLgcF8K^m%^t?Ni*x*p|rT|t98H=D)w_fk|z$nu#BFmI{Vd1 z>x+(E@cfu0RaXW=I{6<}a9DIw@G~IIwx!@wMMp0Bq6gYnVA4+mW>hm@gxISQj)tqASI19TAcy7pt-u)(G48q zZim6HsA04cz(#>4@>>t~L37ZhgFP#;2Tu0EOYsLIvMITOJ@|Rc3ATP;t;P zG<{#W5*c}Tp*}v&#d9EOP3t~(Wx}b09t@Tm=@ELySPZW^x6M61SledN!I0*73c9cxPkNN&v z*YyKl>B)lEb7z>*R!d1aR!>QP;b(&g;*!J}qL3p@%y$Ds=GF@*_se#N&Ds^C;Yfb) z2f_A_3;%d)5dW|Uz0Hn%QxzYn8r4TNsmT9n0Y!6&2yFsd&e@f*N@M;cr&fvA3&F35 zXgnM?-%;EthD_r=zQBV&uGb)Uel7rzw*iB78s;#YEX)nwI1c<2^*;z;uWLU%huT|Y zihJ{`hCa3geLd4P>#)!@eIK|?K+Z0)-7Z>WhaSc>stw^TlUb&(=JA}C-t|GfKAXG2 z)NY=VR76G$PQP=O4HLyQ9i-?pgTX*HzFiV6tvYwK61Y zTQ$0>1p-*>!ZkX!;)@Jm)B*x`%${*WtZOD((6^1Am4YZ->=uCdx$omPjKCiw3 zB@e^(V;g!3Ag~Y0;{C&DT`bpt>!z@KF5VoBfs&5of6Jm#EE2=SSo66(zBy%ns_G<evmSU%o4(T)iEjYBUQY_}*q|3n%0U z%2H%rsY$){@5fUsj}iIqqAK?nCZsd&Lq&3h_VM2SiU^Lh_;}wf^at&2jx%9Ch1c?v zq^X-E%!9MCYrTLQZ11Q|4H!t4bza@L`t*hqB8RB|21u&L43zOGsY?IOi^IMGK+G$X zOy73ddX~fAed6TvzOa;B+3a+^OEJa&N(XbERcA+16L52@N}J=!)t#uW2Wd@CC714Y z(V(u$xMb1LwBe#X_tVgJeFBg3<9f+m#l^?t+>U@D8H$wT!0v?ED9qg}LnouE^8vR$^$zI{IFN`%V)k{dudl1?QIrcW zEG+D&lv!KrZ7*3mYr0y$zV*BcR@e2s zSH*_F=2k9}dj+-$P|}~q#~ga(74+|)UN(In-t-;b(Ngi^>5e^5&v2y;S`*eyQ@+Rk z@|v7mFYoSfn5b%fjDmu-9tS9z0*=f4wjG_0LyaL14?Q1MvBiDX9406AFWhDsOLH3rEGe`!W%O!m;bgz&sAWdQO--~dISCHHzLmG4mZcaZRHM!H2bPT2Oc!W|$(a?+nDW9vsSd&?Q_s6!J9uQ@` ze2e2O%rD#z?`@$&(p`1AZFlZwI3jB;F;s zQMd;;;Gb?sX#$`x6O#rrlRy4mO#9ug@*Ym`SUNRMu$f)|aV~>tQ#Z5xJe#uStL-w$ zUSf8+2@5{@EojtZu5s^s^x!_l_cx_~>gnA_>1)}jpzFXslJ3gPl1N2U2ozt;&DT18 zu@OftCAbulBG>s%U*{wBEW%W-in~fZde%R8N4hL0b)OdnTUJb`J~z|PWTOr_s2`72 zc(@jALQwHHuZ~h;NNDT%ZUX#nnu&qn4)a&#Qax)OhxnN;4F;pkk9>w*j7?)5M{9pI|NOyhd&wNYtvI}0 za)o!wQMN^cU@0a>4jYH$_yYXIB6EB@@3NACkOcReg~cGDfRo+9=v20_$!>r6z`y_j z0Rb@CCZ?uBf0yfaXpN2m$5?SKWnzFp6>M!7C9bSF(sg%JH&u+7xnearG19PF4lGV# z72V=7OH%?zfZ)mf`$N*z^)c(?37R-1z{_yyWQ?J$J*RUgVvJGH{YF&L{u)i5kK2~8 z#&rd7=gTc*u%s4yGw~4)s?AsWp+FA)5}slz_Gc(GmGd#PJG!vV`>q<)s5EV5hxcXX;u`TG6U9#!83&4;Xk}Deno<%c~`ujK6QPz zh*eItdY%(?O5jkPo_U;|xkOb`|FIVjUdgX2W#H;xF`!Gbv%GJ8>)qw&<>1H?fZSpO zlE#5W`&179+M{p9WxRQ0BFdSeDB~eP;0fLS4)RJlEGU~n8P@plX;yEu8hcTpL3ZQ! z$C*sg*igsO1Y=a`z*&Z}d61f*+yme+@Oi6zCIG^0)z_ zg{-)@dXtLC_Vd_a>O{z3Vde^}c(0(4MqzlTqi*GMD#o14`Ok7ef&l?O3#t(R2T>~s z_IU|wwH8M*XSFxZEJAX5P8<}z`~Bliy9HlkTbYa4=xgOMIyEDVqEjpbrl%a4s;m{2 ztu~i~SS#8ENy1b4A_P_pnFdkvE}mHAn;xH1R3~13O926pM7z()IjU%<@0psJ)~sf0 zzeK?24=WVJeDAusjVPo2bO6-Yai(* z>}oXV1Unu#xgC$xeitr=Rbz9yyhUKp9;@Tx;Tq$pXsv2d$&+9ve!eU{iU1k_V*K&b zz?4s;&z`tBb&B(0%5=8FDLMu?nRvo<7Ckk|!>aJAL@W zdAP_@-!}SUq;4|C4)~p_oj2cRcrrK`8O!S8XO9lBaDM55NHg>X3$E<_J9UbF@N>ndGQ#XQ5+6yv9Ye>&HK_Y$MvS3HBUF=kG-`xB>S z3@nt2>`KBCC=4Khp?5UmO}lceNRM&2Zrx#n?kTgLtvD#m%a=uqu;Zq^^I@knTS%#uFwG@5gvh;QCJ*1K738YZl%&y1o?Qd zX{tg{T56)ueb9j(Ym2i)EGO*jcS3=%FvSc&!*sgWd^?Q+U1d7;W_63xM#~Y5d_3}$ za?+!Kur8vVjmM|Gu4^kZK3-KJf&K*a=k+oX-8-Pk$S~TRd>Q{XJVgfE9a8}rOE(Em zDADDNVARD}k%Vo+f{id{%M*V;{JfM^<2|8htPl?UA4!Y_t878~vuMJUEZD!t2U+9% z^Z5w4tC^FPNkRsqG&O%L^@<-~fMjS*UEkXYW_F|O_rtv30;N^F>`YAmq_e!OI@hhv zWIdl0pMoa&LI3WUxhNG1k9|ouM-20m65Hacl1dyw8+_tbc_0`p$qzD6pr$9m9F19v zB~d}b4MFd>T9nTczSG%Ykax9T-SM`)UPgHfF7B-(b0y-d$Io>h@f0?a&6*OXIL1f&YJ4S?7g_Y1;xzhjvgh6t8z;Xe z1NWsIN3;2!SP>_xA{&wQ_!#FGO{RiyC=9ssAEhz;5Ut*u_7-c958>z0NXmGu-&y)Z zT&@=;bM?62^3;62%Uy}vpFl8E0!4m(kF{pj$3DD$zCc8(xl*RIcFT`L8Cn!Cz2~hh zwAf1@!2wHqpS$3LGYUE0NzoU! z{N53N+?h;C8!Xy)R7T}Q!~EjFFRA(5h}e7c76^Y=L0>Mbtr_tOCV|1jK#bw;-E^6+Uhq?S*$=>WEgv_(E=rz2bhg@m$ zX;g{wm;+c*KtlsO$TP4-!P6-`dfyMsY;IN;oXf1}G|+Tb}>qn-%^XIl5m9#V(^l5GQb-hi$)fVL5C$MAtCQl7Ro{ zZoOK2d;}G(|34Aphl3UO!UB7a3C^79cdFpMq2e#Xw1ThbxGIN}8S0v<+y=c6zlUxQ zQW=}CW=;%w(yakVFtDWu6Ro=s)fml7Z?4(AehiO}PV;LwS%vr#Lq!whnKIAM&j)oH zvf+#JGhfe{lVg^7{y&}`2IJ?#X;mvN?F?H?_RK-pOxM8%KE(P3dY>GDGrXi^UVcO z-?jd5iv4d%8Ebf&P{6S@_3-{Vn!x7j$}yTSL(uVII6OW)_MivU%t+_AgFofw{M-Wb z|6}Z}-{N|LzRlnShv4q+?(S|ugKN+LgIj>$?(XivgF|q49VEEB%fL?ZJm39cuj}32 zKVW*!IbGddUG=HE+AMvVx!kSeaEm4q$38jtYp+|Ritn@?&=%lAd(&LHGw} zm*{s1Sgp5x+HALH_5@|t_MiHMd2qNK;il!OF2C)mEvhuZ=@EBsaXlxW_%)wKS@fgL zecJQ~o*AEBW@(tW{s~!8{kFRPC1|vOq2w5n-ns5KOK7Z`j0GLz%T8TVHV zWX`DqrVEL@9I5E*=~II*pk>({xG0KY+K zw9xFdJ>S7`j;fW#-h44MENTj_e0G*teuv!z1(src4a}{`g-ZX+p!SN&N)oR}bnDhS z;Mq5{r+67ZTTTq7Q+xgL!V4qMr*4=1aXbOHQ`aSf_eFijV?Y}?tvqDe)cf-O(B1Mu zyx7<}mc@`-g~h=6UT0B?*WvoY-;Nrjsg|*5B8dep1iWX?j44gT+FIEs3EE_OG&9H@rBheE1M<+_QIA zz2W+ zK8JRs%NccbcYA$_*DCG2vr-Vp>3-V%ITuf66-!sxmENuyjCZPJL?UvKFw#tTv%w%E+4hl9ojzw*y zUy7=jJC+CFH32@Uw@6MFHJY+>eK4Bg{0~An6bCQgLM;>DO7zQw@`R0CX??(6)K`(fvcj0p7g zN~aaiQ2@@{TP!%py|=rYl#(pojfKPD*+q!7N|NLG>^LRNV>TGobkY^vTu^5=xDTTo z^$s+8Uz}G&{#tF+{T9t*2ySJTN&GtDSkwImtI!;LJ2guW?CPh<=>*(Fl7;1O6 zOuvP#XbReStRX5j0!=NVeAO>gH~>g&f-em6K7_Pre&xnOzw| zj#u?J%uxLy*PSnJ_S9$OmXXDcKJ)iXo5No6on=ez>Gd+7y{D$@LMGX;V2ak`cQ4ED zmxMQ6BO1hmA!r(Ft2fTRd3U*-0}|s`tsTY!LJl@opwZyd8{LtK1XlPgIW?GKo4W72 z4N9#UG-1yqOB2=*kY?xCUgmE$7kP$(%%Abp*>1*YDjGBjJP+r1+**mw_m(DE)60IF zUo3|37{cRFN7J-5R7X|3_e_>Nyfrkn8yGJ5f770eRPlEK#bWR8Vl-Qo&fVM%!HIY) z)HCrZsT%w6Ngzj#6ZJf|5p};Tl!|mXu9UG!?Sc=)1A_Nu9}g7_nKBJv$?quRfz9f+ zVtVG%(yp%ddUh57H3OlJ(cROQ`{u^ZXRoOCcH6^zBY(He+NtUa@))!mFJA%othce; zMM%kOT)vtzA50S)p~j{SwnJ!mtnU`Oh+^vH75EL`MxGYTXbu5h-J45`i)-^bEq-SQ zK~Nx4vM3^vy%PP4^NWj@xeI?}umbW!UDl&&++@&6&;ywp?&prRRD+AN@tg8*`qo zN{z?Af>S4c*CRu!Wi>V6>`CcRED7om4*>x_vG?gr`^DQ8UW29Y<5p;_;44~OA2@~L zwV!U4_iaXMOyZ#AM;gEDv4)Bek>1au;AA!JS8|DT zKNx-w!_jnveDDrHP;Kn4OmcN$#hJ zM`F>70@>gYdCkOK8Lv8az9rU0?yhO!pj$U_bic6RKKF1fkB#lJ*lRX(3dk)8;{i9m z#HMeej5hoI+g#E!X28ZyXx)*-)E&-v!RUjnIf2@pdoyyI0b5f%f6=#d#MTW^oV3UtC!g+iz9GqiCTE;%YUEIQ& z-r!kpJZ{>)TRS+!y;Zyh{ua2P-cNo%i;Z0kwNMoK8+Hg(*vHk>s`>@}HkV<)aL5Vr z6+9b=6Wdz{C-MAah3Id^|HgiYIQ`_k`jRZpPY*vhl@xTC0`9DFU#EX#GE-4eadDXm z5(mrrU$-V-1zQoC!K}?mnT%5~1)43UV+y;{} za`L>}HmV;BGdRUzlW)Olh0_GYd(N~C<-@Z&#U-FBKLUZ~j~rS1+245>;sKI!Gl1&w zHogo}^7!Q9|K>j%3Sf&Kv}^tV4H*!JdT~W#QEXv07~Q!O zBanZ3!te9cZPAp z@oNPfe6U&*5!DZpSQjgO3(w)uLZYlWh%b|9%J`@FyEhb_$c`oyt$HenpaLJ^q-P{` zKuh^yW+5G#F)>R?#70R9|41#uwh_`G2dvtW44n2%TchM3m6Q3U@00;183c|`=$run zFnaYO+FvX)V(MVP$=}7LFOo$N3}C8RBA85T`hKU`yr5;kQCCqhXw5lSd$79;w0kPb z%1q47)HY_fm~s665KGACHNGsTrH>M)m_kJ$Y}&9Fv1Z1upO<=#SQadNId`e@NyjBf zlvF(fc=2oJ>A)zB%?!4&L=fDJXo6|tLx3twxfO93k>hQ39j`Qw^os^|Ezi0>5cfcZ zY4|2ZzT@JKu81|QCwKJnk-btNqeimsto_(k14(5S=X93z(n!d-s3F&UGhj^ zd0GJ*GIFWt_m#Js(wL1wCbwjNr^f6IUR#wlQ`S!TZk;QO<@|Fx%@9ekS*^0D8sc1S z(X64nH-UGjcD&4ckItOC%_{zvz_E%UrPuvRm66B+DvMMjUj`)S__m%a9R^OAh>P+0yj+7*=FV0V{dsu~>_`iLeUk?LNH$0y&WsG=$2rQ+YFzeD6XFu#LvBPTsTpLFPy(rBh`~{l6R} zB>g`xgwxzEM&EKr4eiZICqCaotE8LYr#%znWE4)Q2*zS-6SvIOa`kK3l>1J6UiJ@A zXBfEUcBre|1*f&=tiseIfBveBnPn@Jdr^*76jaU~%)7E0jQEVK_E79{Of!+E&fz8q zAi6g(?Cb051a5()H@YdO0)8I3y~cJ94%WP`Ybl9F9G1U-g=;u=+=c#)BK<*zyLT9y zV8~!UL)umxR0PDF3MTn)z;zIe6U#**CcJT6})Mn))f`7hPxfD-?xG3?MN zwr^)#n5LDKmu~?;BlQuZ@yQD~hR)GTJ1eceP@XEy3vM7Lk@1NMcm7k?(wJTzZSC0Z z=eCPPD_#mT_~Ztm?~g(xupi)|#j}3Qd*72e`sHgmZ&G|#qNCJ{cJq++4c+cnJ_f33 z0*49Qo)uTv1rF$drVG|hLHB@h()6L9>sFIXgN_u8pI7wYJI^G%2nN}98?gGVl{H?5 z0q{E2sK}n8UIl4llvD|GY`YV5ng#Rn5Df?@cW%b426x;Dm>iZp%F(T=IMvuD;nM_r z!jMg6JWK*yGu!z0x{9kvY-P~LT>xcJ5uT=9v*4i{pi#E9wax2`f{U4o_7Dw8R{$J$ zcnEg_&>vv5P*#^>cXB!XCu4yG3ll35HhH0P15sBV45#rmycpRD74dmU&d-PNkV02$ zd-*RGP|EXq9Y9lG&sZoED**V$^Pqnj4A%4dMoYJ!pM16rIx(9}Yq+duaOilfKfx!k z6}*`#y`nVfVZIt8 zgzqGh27ge#Q##raH*Yf%8@T{32CvSIc*_NT)KMKRO$Y0rsNRU=JTg>4<*OwHO9bkg zjytW&QUN%9Wc%n;`?!s=3toNVrAw&D{1D_}F2jlXrq>=EzcoLe2wUlu`pr(N22dj( zf;(i&wm$s{SbqmCv`sY&F6Qk7*`EDPK{N39m)cv60)?fBKkn z4P^xSP(QsXD9Lsc+&+uJH4^4exXePN0!5PM1G6b1(xGDLkWh-HdTR)Yan5lqQ}ht> z2H8=xPWmJa9bt>RAm#E_Ge&o=fyqow^LkvLCT7TZk0X`4|6thKv|^RLY}c~MmlX@! z^K*@Exjtn}7#h^;3xE7j(q%Tiqi`~>qDRkf@S|FQhDYU@Ld(`+U|0=CYJgfJWyG8s zRf%kh1%1exJvCW7IVmM`D@a!t+#(_XrwQ_4P$CEc0MyIW(b`u-lhmFXZDjb3oF9iQx`*@PLqEGe=h9hgAY`{zyI~@-}R0%^(m&I&SdOm~Uh;1KEuJ2a83-gS1 zSrf0Y(GLoz>l?&;;O6F7+s5Xm#H*eTbeNoUNFEPvWK!pJVC)N<;z~I8U#l4wo#!vh z6!ZtzATEP&)oh%95t5Qo3Rl4fl~)hM|Gt6vT-vNIlmzQy@`bpgY3g%|BY;=k*{L&$ zk`-TxNk};sRVgi59N{zw8Zr``)*-Z@P(-l`L~tvP6JE9DOf$w6GAwGb6J+jaU(i*x z0r2i<+E9t$^vqv4I6aClv_;qng+nNG6ZHr(`)gMgvJy|IEkwmhbe@rC6y_t+*=ayr zU3)1yZM(eyfmu{_HfrLNWwG~&gPIrf{m8Xrn@h*TOiG<8u>eI_4VKPx1&sttRd5cE z?V`$pTc7!r0Qb-gl#!{g%HdEzGW2k|4E^po5^<8pRoq$adGpM2qMS`gO#GuH(%b*( z{Td;nznk6VKG0=Sf9o4kEy%{=Y=i;|mX;`Jpl%Te-H;g+za zE3RkP1au{>YicvuWQs{+V*BmdFiu+zIUFaI_@*gMnF9zqSWIekw_LC*<&B2Xwz}Cw6)_KRi5$+D6QnH}CD= zbm`V|{MdEcDHU~}nw$XRu+EOHgBCF#rwV4BNfli*pa2T;GB&PWs}@y0yR_R&`_k8& z)d9ei9Pc`24+75OEccncf2Ts&q1p*LT*Z~PRcU4Gh{H62tKJ@vb%ftiyma9=8W-1U zJHPtnrHUfeG1&r%&6T6iA?~;Ab7x#6>TE434Ifj%@!(-nSPipOs1TGQ+%$22x&sr> zoQp%ZY=`DbCCB1E0z0AnmcbB90VE<(__?K{V{2#J9gS)iS=Ai!%DWh8IKnj8k=rmY zRno?ah@Uyi+TMxlWw8?F$%5;ocg@u^8w|^Y6Qs$>(u(M_QUDu5Sc6kN%mYAtVVqvI zRxJ-#wj02;py6-;_wd_2^eCFWO|y^34D4v!xit76hd+QV`pVmQy>vG7(L!Pu(+=H5xrj zZQG8_1C!7LhDRTf`=(XX{xj&XhYGxFh)Tj87t5M{o7UGB*me3kA)P1&Cu7VirOcx6 z?Nmaw1_F=6gX{2eu^GA57r$YpYUw0DgX+Pf`>;#N z67A3*7a>)}rQRMs1hFie5?_2is>m7-_d2JFz`2#nGLin4oBy|$EqmY3`9)-8ubDMH5a;ZLv^{^unecn z1nas++dRT?bxFC~G;uy(M#VEekIE1xSx`_;+p5?ydYEqHb}sF2qwju;J#8%0jHR&Xx^1O7C3>~<%V@>idkF59 z=;yb9U+os)zK4Tp=(I3#gTn}{->L+!8zTNjD{vGg8eiy~rrYBbuv1=uUK|`ez{C_i zjYhu9P&)UvKQ`qLrsL$WS#3Sa&1EVtB_~CXkBrAhk0V@7Evz#TvGX`!t~ zY2yA6$dKgx`kAP&%Ouroxwx6^t2tdb`WyVM$5n`BD3wC0#iC`pE38nXB3dCLsnq4p zucZ_BR#1%C=FdJhJDCzpyhlL~nZiE!6F}r-QgIN4!cV=>5ma^PdcDx-Po?W8xNf|K zvc8NGC|Vzjzw&2kqiS?C&7dSNW)*W;9^Fd1aQcC(rpfMKOweA5TQ<)xd)de63i&E> zB;z1TM$L{*)@1-T^#E+d)A_?NTEh1`uuCnt)h^m)1{4?oD zlsCL_NxXuOxLTqSA5Mfts$Pu8hPGUgfd+Uf&m){=qHcEjsAsx}sbAtUGkNOk@zIX9 zF036v*EgtntyBQul93-$wPC{nBO#wN=c|C{>m`_l=5#_NQV33_Ty))S2d5h9YZ-oI zB3x-E^nlrT+R%R&z9^_fM2OrnvXTMn*S?yqj`NYjV##V~fCy^e%z6u4=ou=(gt2`$ zJ}|Xyqt&(Vsu1q5C7ajpFrDT+G?#&?_VnU}&-Gfz*b^cD`D-&G@!w+G?I`+8r3%Nf zUTW{W1>!jedifvh9Makg81m(jYG`h$OEvr&k~RUiw7qEP!gde6A~Qf$5e9pw72nHR z651%jXt@fvx`-^!rCS{)hGV=LcNFjJD`IKMfgJIE4SUo2rR6ijcA-UOrNEv67V`)v z9kDzj@;OEHBZw3WCg>5^KF$HE;#NKw=F-CAJ5JBpk1U=~!6@zPM~#u))5ffA>rn~A z_&;lrP=@VuOrBtZeLIExx?@j16^?KUa?1K2Jbs*WcsxhVfA@cXtOZ9eNYLn_7HrN= z&R|EsD|9gCZ(FRXt7@9I6xx@gm&dR}x&1Plk%z;B$9TtI2t1_)V>*u%_ya;5(bVIs z7>-pdqEzi+Ak*KjijL7rbZ&>{+VM2-wzij4i=)Cm@{+EsuMBoz=dZ{7559uZGZ^7x z%nMLIbIYh-KtjMFysy6GyR{-6rH6_;%>`;=X^zvzHA4t2+=j=~Q%cGlesqcRd>Yk( zA+&!*r9eOx-Cx%$3>tq-0e?7&9P2YQdb&DVT@WVa*%mb${)E^ih39i4e~PGizn$a1 z$4W}bv@6G4sL_z4P!%MdCHai2sUA-REq>qnZjkYNTonXzePP6d<4W`x>X{O)J;QHEH-VHv{0 znL_~o_qLIGb8kU9=L8PO-VYjnKT#ioaC^76S?=;wA7^=DSnm!$gn(as#2C^a{s)@u zXEY?r6NuS#Oi~JU+2R1n)*xleD$^m48Zxwq4{(%3_hkEA3-(XN5gcHO#E(@`E3g{x zip=m#5oS_V&Q>EL!mv;X4LNyzF$*>Nph_imsSd2<1K&y-YNj;)qUcW+_J= zeQ)s>iO=-n#D_>=S<~NLt^zJvUHXHRYDX{TFQ|Ae^KXHb9Ea(=7{4yrLYnWz^L8x~j<4b?B&^s1Q97qEDPvD)xM+$R z?#N2QCg2;Qyi_XS%4Ct^!uO$tGZ93ezsu76%aww~i@yRxP(C5oms|#cExst;@MGkB z5{^Y7RdDDR9rOU$*jc!*W(6aL+zHJTEp;)2B~NkOdQ*jj*4gq&m(>~&r?&8t&wue9 zf4^J*$Imfs`36_nGeXuU(!O`Qei@Y4V|eqUD0M zk_-%xPE|)r&-^aejzb2;I9NPr9Q3Wv(Gs^znZg7C*N0u3CQt zVwYSNE}9*v-OD1X)8oyoa-e+WO6w&+EXNdYRM4p)xBAA3XBKuO=n=W-7hW(}q%vZ1 z`{==6&4l377CKobQ$+ROc!7ELT^N7AT3i-uzJ~g2T9Oiv`GeZCJ{<>i!f%NYYh8&5 zGQQq~q$KkMK$O?icY3}5xfTShC_aOl?fD+ixDNAS5Y78`DTcY?mx@dHuoj@8!_EcSr3Da(*5!) z)y-GUC~~x!puh7#9$W`B07pC#M+gxIK!*2J<}z|Uw(}CRkUzlWR48@I@agMrfMl^5 zA00>kkHO!X(T%%mk-qdyVO)a4Hc6uD?Dac;j*w^vokrfpNdtANwer0FGnQqsLK`~h z`VvZ%BsNVXdBT-ES`ET0TBTJ{g;6`$A2IQ&T(q7T%2m2N**s+8Kz}+hpP2HgDhx>b zyI@&@|3<-8TOlRT%fu+fDT|~0@sUYR&S094mhRR98iM~>jJ=~GVq1|ZOi%0A*VN^m z&$FsZrtFnPQG8qoaY|CfMc#t&S*4I+W5efBvL^oxSteF!{}EDGW4Y{Z*)xesGyFJ_ zs+?F-J`)3;N`r97ShLMXn&0}Tx@#Gm+^n16Ckchm9;$^(FhyGCLo^5+lO=(oMG*fx zxsq+)7fbq&V2&ZOlLNy($ioq<78S9{u{KVNRNbSKH#$qX%KkK>9O;h_QQ(ME3P$Ck z21H1?`4q7B*`o~a2FeQk_f)j1n;(c)XXuGpL?t66FbClAD8CrQ`B^f4#OF{vc2k;3 z2!T2%5BsszXI`Y)#-tOzPy#uvj`icip&*v0BnqLryktbc|E{fJ@nk4~wG6|6Q6Bw# zTQ3fId66vSz)Gg95;}RAX(4Me3R*F+$CI9%2|0nYB3BrpphxcR!(QUA#Dp0_l1v0y zeb*i6mW2o#Yp8gvc=kEI50_6xAXr zcXkYzXfIgVDQi{jGIi%{1pY!pdY_T8j*#)EFw77C`=QX_q8|r{8i#Pi3i<-*zQKu` z%;W40Ck2X9!rW!BPV;JutF6b;uf8$dhklMc7Ed-^1#fNFVELqE#J@EO-W$Wd!-BRE z_0kG{H>iJ@efg2FFO6BG5Y{O3WdsYCCF14WY$kw_;iI^?Bn>yyto=*vp+rKCC6*)- zX=a2VYmO^DIy3lK`fujYAfurNQ*pObJv0k@tGEqfl&m;yT=}D=E4!$OAs>By4S^lx z*C42z0I}N5+~FEXFk%RqlDvcqHzl#SA;VR=Q@S>IMQT+B>spVoHbD3HNuL4n`}aP9 z!^>x<`P50$ypmN${z@q;PU|~<#Y(w!(~$#I%u|Q=o}cdDcza>Ioz?8PkQG68nUkh- zcu{&xRf~yP&+NY;D_v`G9@>0^K!)CH6|NsCB0m!Pm{TbuoOoARqKi9LKc5n2=S%}J z8Jao_yNC8jjvtUhZ?z083lvvgB2VXag0k_ewVYuU+}6;Fb7K$Zk#4<1U@@&5Eb>0W zlynW0OrgyC;^)k3v#{QQ47-7QL2@F3e@2Q@Q2+UiV59Esw`sb+bAz(zp)yQSeP(x{ zqox11A%@|Ub2dlq7{;mh{~AhF4#mR8cX4t3M^}3zg^;CVMGAsi)3b(|wSIZ-G%#T1 z#fI8O#Gadihmln!JmJ)dpNPw>M%<0n94Ez17cHK|7m`cLi%e!g2m)L% z!GnZN?l%1$l1l||!jA5x3LWQ>k6wb?L`RT}64!{)DPM%rpFM2czSbEZOU&6hmMX-z zp`ysTxi{AdQ@z1cAF2N!G5@!<1cplDYJN|O&^Uzvmj?GIViM>pGX=>4aE$_n9`wHs2ytrpYQ46inlb1T*CjR~)0%WwJBO^Snk z+H-z8;-!`Z6AD?Ynl;=t!orT7oWACe%F;Zq0?3mp%t$jke>lL#-I|*GrFnwU7g^!n zD!WklYFgGJ-2zlv0`k(Dh~M#-wUj&_+1nKvMt)vvmy#MxP+u-r^UqX*4QQK6&9%DU z>Ddys=N7w;R0_7BR`GKEdL@dNpke%*b)1cG&Igj0^ynulTP@jN!k#P)__T`eEwVn~ zfXOhe%N5B@tR!IzWXzs*T;~CE*IewVYjNE9ljW7_;u&3|TZ^&e2xi8c4i&DWltU3J z9brNu!;uo~Z+noo{|CF4Sj|Gm%A2JecOAO-?Updq>tv)Z158jGo70Dpk1N8G>bUeC z$f#stWaH05`I2vAAasXMT((%vO;NSDKIxIHhw(eo=7ly4&0wW91hRndcy|Gq6yu=o zXwRKE71^G3TxCBenQl^A!mIimkR&EO?(Ms=Mp$s3m@kFw=mOye@40UlN-Q1i8V_v_>q7FsV>d`(ES2(Fi4M7f!EA9cFd4U^JBI zl>!1a2e*NIJ?&Uzh!Q`i^bxyo$MWJD^jFTWzVOmEqK-=8U~(9LumQ1^Q!5{Xep8b@ zJsn=H5%^^q$9Dn6UrME=$0eBP891y)=s~REOX>tVA(%s@=NhQ204dhH2CSpm3Y>ox z{P56$(8!`gFvVU0Ffg#SLuV={LFFcg#<(|#NTk%{wen(ZHH0?d^RIS6ODA$0YAFa+ zV&U*e*-IzEWH4a>GLfY}S03zn#t*v({S_^B`!&C;R9Cu2n7)IFm^MOUCRj0_artov z*8(Mz^x`2idm2@8OJG%Soy+&FcWIJizV}PfmbdUd4PK_I0oBy55fXrwkuc?6qSo;^ zZZ=V*maK+s7Gx)2P@BFm@8^&r&q$(wq(6j!Rj0qCKbMkX{%`z5mLrRv9Y_9@Tu5ea z&DbG0l0iEDI9Pd3qD1i#5ZTyc+_s5HVi^3RU@9Azt%(N&`5oCUw?ZZvO#4DX(FQ$h zNBXH?(SphTX`_N~m(HDsFKdyWL7*vNwD(YQ-iG}85Vfm4J*%Uo?x)h0Yq>2c`tr^4 zc>E{LQfaOAP`YoS83CT7N!2%&qGND>8EV>8@RpR&hmAflDV18M8j90Z6?zGfzJ0&bY++E!cZ^*pXBW z*I3lxCS3swUg8oX4|qD&`c)s9n#h?x%VEmRGE12}ZkW!EEdb}6$}bO@j#;G;g;uK; z6_4l9hQ1Cn{G!|b;Vsj z6V}J=yBAIFvLAy3h5Z>O6KZ~)pCiwU((prHZN`+Km=f@Bt_8ThD!YXwlVLObt#<<- zvhmG%%obn5nDkauGL=?uH_UTUkiMqq)?wr0bE%=+4HmEDg}cD2sV#>inMxX@0ebA_ z&9!2aV_gkAv%Pmyv#v-Mr;}J<>wuPCO&MHp9 z=zv#{oB3zqe+awM$oP8c!mbYXAyY1HuG>4e8%nRko9(T!sVRJp{N=P_5!b2)UFXX0 z2QG5-P2@12KE1FJ8-E04TpO$54@xJ2Z*!mi{OGD_PPE7`C)Ve!r&p<6YHUIRH(Woy zxgJc_5yZx_imhNCAK~e1_u>yxCO*N({r0`AM?H@$-IP9fKovg;L(hEQP(;oeodN>^ zp?)xy#AI@oL}bxP&$8Eo{V*T)`VIP|i~4E6*hjuwlS$q*3OQO?LoY{zkAKs6zNrN% zhj(`WhFu=5T3O?>hN`Z?@(Q=AcgGxE_a}m%8%_Vk0_4?M$=j>Q9j&KVbT)TfWV*p% z!ItI!S)2c#__wqP%TS8ZW>@@xkX?0HrSTorNSx2_+c*kxPXbHb`v8wT)_V4XbSvh) z&iCP&7&zibB-x#2;Qn}yP{Kp042gcHl?q~9c;h9xhCHNfbXZhP`}MH>!DPOiOaytCm=u3B56)g?G;^Hc$bW?}N zJhr5?R17w`k@Ohg^Y+eMNo#@h0C+%tzlpZjcjt0;3`yyb@%MWxL)dBgun{K-=OI`11+Mf(Za;v+&mjYc< zREmt8TGuLt1#h>2?EyZ}eU;3~a&>igWle|A|2OCjxpp~JDBs`EI{hg z_896|g!6@>(mD6J4`CxWhRFW?4wMV~PWAaCCL>6FofOeDGWtqG{^EIIh0jwrpP^|; z!0%x!R^t7KR-BQr+f^7l!IKJDsk`>c7UjYo)_Is=v}8|^`^Sw;9)?1v??{1({uk!;T6wJ zVT1sQpXsYoKuF4`=ZT|u1087Wb^WYS5Ftm!!{D{XzdGEpB2V1ls-lh=#+sQhjtZl^D(Z>ph zNudWO4txv6Ae40^qwSj~C~YL|FFc=)IYi4s9xVd*{_)>-e^2naRGzn5p{qg~AuRfS z?Hhmk-0ky9KAXWl?RasH4;gSrn@<4nI)mzN|It|DA=vNiqJ#Q&yLB@rGhz*Cg8O(R zaR3Iw<%`i&y#V;nB5bV;Z-hKShe%{!88n@puWLJxlxaKeju9n-psI`%#^_mEw_#~t zUxG_7dzvAwnD}Zdhh1JIjL8nZHN1yo2E@;RO_chcyI86TW9 zwJZD8Z+)|OL47S9sCu8Mwo)l)o@mNAP^@|N8Dk44!8g*?BxD`;c|7aY);?p$b46YE zn%q53@JwXba_`w#lBLEuKM71xfP(y|vOxJsY3$>WZl$hs@&_YQ6dd;%?mzhPEp7h-;PrO# zmfX4Chx)oHMIo~{kt0G+MNj9Tfl`Q)^jR`;MKVw#levO?N$|- z>)$fCcm2hEEPae6Q%}>bl2{wCE6T9>;*kPRxUt#hs67`Cs)mA!VT?36dfKUdIVbe+ zo&0_5*cGQ|s~{zHqB(1P6sYNTCI}jlfjiXBin$~^z%dZNH6>J@s`bB25Ew z(R}NGDZDXUU1g~LV`Fm+I+I{H(%zq}Z9CkiQ(bPCkv@=xcvMezV`@lQH1OQ9tJZ>2 zqVxAZ&NPDyha_w1VfWRNIt)p7VL*%sAG~idBJ>62W@0lEymWeFpWL+-AE@~37-drV zS={RuGRWYFnyepFICeKqm~y@C=HFCQAf__sUS8?9uPL*LPRnM9!RXD>(^tTY_3gkF z_R7BDkFpRf`n4Kgtz<6wBU8FeO_4O2TlwFfpUA|L5+FgwOR^q;nvyC}qp1r=%;%49 zok?DINOB+$>U>{f+Wl`O76-Ps(Yt+b)o2@=f*P!uaUhM1`m-&7~gxsh1qU@1W?xYjsn452JJ(pO?qq`Q+1 zF!i(VW?*q=I!}B}!x}1Kq)v5zyw*MI_Q~lnN9lziPYleow9G`lWcZMeo7QNg;r(Gd zd+N8DoPkguNtG}>GW~blc=HE_i!N6P&c>>hZ`&PGW&w()$`@}G_O}c?hRRX%pRoE% z^y083=kB#+yg~h2_Z!3l4{g0=42X=H{BFOp*J!_jv}tNOU7KgDd2Ls4(Ss0(aQiFR zVD>1&bvuDP^ITl_;`!CBsQ8-3vZAv}=1} zB#1yg&}MlJ4qwY}zy3ru+LQxxCw$lYHvRCL6T}e_=zOm49deh9ygQ(VoRIqVnae$I zHub-C7r|Fn&`}1v8WB3R3tl=N?DJczvSlJS9K^16Xmtw7@O`c_U2q!B%7=_5cRnMB zEu67y7NsEmyBSk+SdQcJ`2Ftb={cU#js}B`W)i4yW9&CNt}aa~f;heJPmeR${JP&k z4;#o6JIy+bI7Z@xEk9je+&aZXdX?PEhbgA*%N!qHU2N__k5@ajUC%bOuLs@lPsb;< zd6$##8##PtgH)-8&kecn*C5mlcRmBJ&bJ7L+CUNemHMw9*0`&BgV6;3ua9*q!cYAN zHx?Ea)xS>bD=O5Qnwkg}{ec@BC}?4%6+|?w_^$5nam#r-n=JO(9PR3_`w{W2pJ-yf z|E^gtZErs#)teNFUDY<3z~4ia`^`ELNOgdE`*6Nmm)}shkjG&Sm2Dp7OQ6N!r!KKB zH=BXsW#j&bu5&VPu2hkNHGonj-olTM*!yIKaXwDT_Tl@q8FwVzw$K-p`R0V|Haac} zd|)^Xv_?gE)Z+4DWEGtcr6_9Q%Mbie7Da0NVV!y>86r^Wp2DCKo{_qL+d-t|B=oQ2H{=@-wcqp%F$! z@yZJXrW;x$KS@zk{l$bNH&WJAWz{p}^2=r6aJgEpfK@5fwMj{Pftu1-X7TAZ?IWYy z<7rA@W}z2wg-!Ck99*L9(}t6AHK@W2vRP0=<7I(p`*|=BIVJn{#cA$h^##!CD86 z;O&-sekA9y9(vjVK|h2U_xoPSl?sdRXWrpO15eJe>s&>5g7fc`#U+k40-bu#ll92q zRaT%){eA{qQL%xT8aG`dm)hCuy|D#<0Xev|&LfW9_lQxUxz`xbnuYSh3i47`_H&Y3 znKt&9k;$Ao$=|H%h1ZA}u$h=(xbzaLiXH%XNem3n^z}hypPlkS!WPmojBVdM!@|Af zA*7_n&e}8~BR92r{*XiK8alBU=MKIIJ-?ie8`=SPh9Jk{eSbxC=-NwZ({SbEYlb6v zMQ;4o6A9*Klmc5EyKiEE`B!-!{b7GGb7)qzME`$CcRcGpE{C})!dC*$WMRs+G@Cb> za3qw-^wVM2^K(<83@D(hR70ODaCH@8oQlO~s;@YnZ@~w~I>V9Enoy9@*DcnOY8xIq zuHd~OJ*rS9TG;EhY{Z&e=>Bmh*8d?zpT2T9Iq*+GBnz0J*X;+2=21{6kG>NS5a@pB zybupU79~w0GlU9TGU90{>TmXy=^Ko8MMtLRlo)nGW8&l6=W_}3Q$8y{U)39Uc=;`@ z_4x!4vM#{$4AlzUbA#;C=p3V$2E*%~HRk(^BYHHv165krmEAe50!n#~bA=Vrf>TMDVloj?E$pkgm+?x$k*syUyU|X*eJy zH3zsnLrjeJ{wI39*&+d$i$0#3f3TBYol5iKe!BELpAQP)FzU!Fp#_8_;=DJWTYqa! zV2WkUP8b-N-TPEAgq6*^#_xvl*xBYg-{VH&uS*!QkL#7r`W8P2eiDOhAKLTQo;Kiie^#^~|M<9tmaD4#+ z98)@0{g22e=yf}s#phx*p2lW1NoY;|1WHu#ySZC(YZqc*upOB#CJP!(C|1Y=1AeCw z5-cXYp36(YV21}pM8t=S&F<%$DQ~a#BbHnNM?-)AcQBk!+t9BUOu}Qr9)s%kyBuTb zK0V&XGX`A=i3EUed~*ye?=9`6Rx2x3x$ftlo83DxOkh3)uo-!Hm*(MBvUH9}#LwsM zV5^VJrovXV2W|TsuQ-khPd0V%aR$D%>PLh3l_~#bGvNJ$$m^f1hN*OJ`_&H5o41F_ z_A`6?ckcs#@S<%ytsjewh&b=KVYaq_D3?b-HnxDgl8-q)e#K&-2A-Y!zq#10AD!cGv~oNP1*S4plC@;(-W1n+f7Yl=O3d6`TKd|sFFqEbWp!w6>tJ7FwT=K3 z=Q5+ocHlj_^Egz|mC7A8RzICi^6=T;ZlGpN<3>Qpd|0{=9`qOPU)N^AjaR7}^Vx}9 z?+m)uO6jP#Ngc^+f2<0h8&nN>I@sJKAEQr|Lug_*GJ#dAnVJ6U3INbq3fXR6RZD= zuD1+|vkTUCk>DQO-6b$caCavVEVvWggS)#+a3{FC6Wrb1-CfR;cYphw`hJ{0)D#6( zGi%nfR`=cA*S)_O5iL-2{N(KgqXST}3OA>Z>Sg>g^0ZkTqqe2QEy0r11oIinF0$F7 z?yuW)h)*r|G1a{sKb9Ug?xG9_%fB9H-(jrp*#EM0-7mFCy}26}VMU~;aUb-I{`YSI;Pk5D zJWh!-S@w23qg)i_y2@mxJcS=oBu{h{7!)P{Gu24PYb&yG-Kuu)-qB?)lhd+(Ph}xO zz;IkRWBn2sbt~{D<5_W@b838g^)idVB_MI8=B%=LnRZ~xWOjvK%LeIBBOYe*YXfQo ztmv>l2>0Wz5wR}uK+mMz_G`#t;`6>bVlNhx z)~CP;HMF8SLS);$IW-~w;ZSsH9rv^E^CR_|XI71C^NuZO!@mKf=1$fJ;s&$%bpGAR zf`ey`=gPdN`@`v4Yh}Z|(I$Fnht>@yn`SzfX%s-0$d`(r=+xuF$aFu~c+Qz&XBX98 zTpw3wUEf%^sNxybFu7hp1&~MaS*wtNK*Vfy*5brbI<|A=apB$=*N+oU`2vVY85~Y{ zf+Q-uWW{p^0`FI3<*GN!_^mB1LFDTK-a3h6;*K00&-?is#Gb9MZ8sqrqw>iwD|hhn z(1;SXLkUnT0uJNcMMbT>Mj$nnD1PTYw`-am$Kkk|urkY&5;=(@g(Gsl#FAo9MING7 z_v#dZ`r|0fK9&5P$P8~+Q^*Ux%OJ6t&`5Ir4UgMeISF3^MD8<%3hk<2R7z8`98L%7 za)De)?wz$&B-AIpHk4uo|jEJ1%^)rPPUedNvAC55@S6!#^Ok;gLH8&sAMFgt^lP-(#ZAz}6->FPR z=KJ|c>!3SRi^1@mZ!cX-Yj6;>*7rPwByHyI^aAgzB_tvj(Q?Dx{-Bo3v5icI(m~ws z%8vI?=q96hPZy>O#H$Ms+re~}h9#!i@q0UvFF)qJf4EP{Z?C_%rCH433M)#L~1%5YHHnL`y2URo0l42iY>2txZK8RG#RGe?Cou)iMUKc7j zv@zvB8hh>|fsv82vhDV#+6jJgsM;l9*4Lt5BZnR8(|8hfWtQjo(m}cd z(Oc~K`MCgqt6f!Fd#QETAGq%FeY5C%3i4Vrqk@K5O~0vL(`m#_k)!I*PrN(nEbww_ zK0Y}vlKb7@y|F$yon$BxE$lC?L$oZ8o&$S@XPVB|oL|@ihR0D;bE~E;rZP`XUy>2{ zrPFfw3O%FMg~dcO!-ZcT*4Op5v@&;N-1vGI4RcR)ezC54-CFIH(+wg+Crekc&2*q} zzmWTJ;)wNNE+Rw4WxIF+`(0~nafq?#M2m^%t)#6>nbEhX@QN?dwb)Z%+Tiu8q2D|lR%3&H`3ZR{T1`-Az~?F*KvEDd!ztwP)N*t~xv9%8HK z-%}YZ`HsCF4c#+NE;EH4Mh8|8b|a5qp7rMx?lkM6wyDl0hoMEKz>$5m&epgpbX{kn zoQg;}&*XF&%WtM*h2t=IJS~^U;YAg%&yr@+O`=Td!lzZ>^dao{2?Dq>8Ip2seb%-c zD~2DZ?Qi+3>I;w`;&T}CsT$UxBZ(*`lIUaYo29LEts5wMs`!-w7bip_f$0vtAsm_q zM=3tA?9OD(0GFpbR)vKJuli#%XT|VMNi0c(S~H%B_tV(DsWRCCC8IMKtuz z#k-xXbJBimZ)V#|m zJ=g0vyl|8I0^*Ou?H!(R^4f?5NgBu~c14i+sllq)%9;c#*5mHTm&+H@UK)bh>#6O1 zi?oPxMTMnB?=*kovU9%j-U5Q^=rHVA=crIuts- zqsSeW0x6NBDBeafmDL+ACR#_rEQs;zh8|n9c7XNV9AV{)O8^>b-jtTA_-WSr6^u>5 zD1}wI+z*a&?KaD&;;EfOXw1ri+UpsY!<$xj_ZIoD^HF7GxB}+SQRRJRhDnsIHGldK zZV0)n5CB!BnW7cK(%QW6!rSSXf<41yRlO&>#(2cUtd53;Mjf~t-7nf(jgG!O&Wva9 zP!jd8%dIS1=-GDdIi}W*=J3I&t1~2Jt-D{OH#?L{sKu`dVD0or=~f#+y-6F2KrUp1 zY}Q|&ZmpO?mr*j7Du08Ac`+c}J>4IDxz#(h4S?b^z6e`s)YMD#P``tl2YBw5;- z7ia3!o?E}d(#C?!`_Rk=m+Pl&!`%Fa#2I&rntJ+g-8!2GtehgS4dqk`w`fNhq_uxg z#ZLDIoyt<1Gr*wd3k)Hz#EDjm?KTp2I(T4x>(4BAK#`wFm+C7K*7g(uL`W!ssLEAG zL_C!`&9O2#;}hov-#W71NE)E8IoD&v=B=USM+J9YaTm9ZDnMGCf=>HUTNEAOeak}~ zP4DY{!jTnctVl~co(ak%G@RJmq6_{@~H6>>&1~Y&`K+w48u`Bp!Z`+A? zc}R>Ntw(8HNqezrGbAE87Mi$>QMox2w0#`Xn{2;+kZU5I_>%9;gL|0B4Jz;g{pu~J zPt>za?8?ZLDHoz1RGvnc zI|PJeC??JVAjqLP^#u`ljil`A&S>W1l(C?-AIB~>l*=nmflE1AK1O)*j(m@KE@8u# z&CMEPk)C*r4X^eF1_sdqZ$v7h!gxDFUyIA$o>_x4G8%7g5Dqe&8U~evQO+}6=4-C6 z|72}^cvAxmxnco3AWQ35hxLar-b=L2pFXM*nSp(C`+=Y&kf;`st9yEAW=zPB5+X*o z0zh)V0au6hHd~~)`}Se?7O6@H32I63+|PoK8I#)Bo-REBSe(!NAFnsRty|28KKEop zjL}_g|4F3xo$i_k;LaUfg(iq~K)f?0apzP74Grz}a%gx9V5L_cu1CJiRNmg-+3Px8 zv_A;DyxgAwOeKB5^H?TDRvaVR=da$NiWVq007HJh|0Y}b6B+KW_IIKE$Y}m5nV^^Tw~x)L;}di#|FUc>L2p4 z_v6{mrC@NW_NO(nAa6LpKhb*uU8B7gaX@ z`lBlIcpPpM{cz35&*_c!U8oCRP@HV2EH1x#maDrrbw%^}Xo0ZzJYFJ{uo)02P6g+* z-2Jq_9PF7gawn_1@7_JCc@8c@)^;S}sdL(A*Kc94ShG7ruv3?GFd0&lkzg2Z&wl%{ z8F0{G@~7Tps4&gwNo;8n9wKB+)2T`yF0jlJ^5hCCyfj=U#QM~?eQc70{JlKgV_O6u zv#$IMsRUZm)C++=M1V(};FaJ^{3P?gS^(BuMt;K9=KH*IQApg+ZBan^`#!?dRMBD5 zA}*3}ZqdQHJo0tZYy4?vs72wC^SLjs8>@9*{O4}bPuZf@_G#lv$P0)q<0A^Iu?#8btyt4FoY&gd7e#pMe#A|G$Xj?J)= z-M`8Mlcd;|KuES!YbfLGjdHSGtYr8XiLRWnNZ0kMGJK-h7NC{Pn54sfY(A2(m3lm$ zmy(%aUVnOd-?Y{yeTLkLYj%CYDeSiUX~-#{n|w9-t_@%#so~zA>;E^*TnCM=v1YsnVfGA%6Z$)BFj(J7pq23Zvb9&m|6aVqgUYSA>I2OD!xe8x2M+oh5lhd z*>^_c*6q#pAmd{%S%rqF?J;+=vPiJdC#XF^DCbv%p*;cTq}z;~&1B{4>pJD`P?*c= zRa7rpCM$UG^tg^VV#)0tPlJsOs@vAC6y;Yfg;T3`!vS+0Ko+577bl-0#;P9flXp>j zDNeInnqSQk%H@ksR;WuPME;4{FqZT~>syE2*+EUFuo1~T!_G(Rj>lmS)$MycAhhy%|Iqt=fPd?$Wu)-LNwnzKKnGQY zkXUK=I9Iq?{v8r$!nJ6?L9b_ef{QBZ2ulXQ)mx7~6XYF5@5{G}yMx~hAvcBm6&(^R>= zrQ}==CMng>l>tjE6c1KG8hK5d*F-h5W}s(-ha>B!4esnQo3d%r7j9FAC!iY2Xf&y? zpzN=rwr1P^p|i%ac%fxC=9m|1_MKy{cAyfdNwQj*Kw%Itp{!*ajDvWuXDNStP5&dX zG`GTMS)DqbqRj4!S3L+u$7Nn5d=c(#U z#n9A*K_<4&U@LJvz6E-22OJ%dbzIr)!h(8_mg+*=@gCDIMkx+b&sOcyEEIh+@{C79 zfD!K%F>Z;UBr&_@9ZtZ_hV&Jwbrfr7^{-(Rp=Dt6%eF;vZ0YjOtd?)2F7fZq2(t<; zyPZ>)_g1U!B||H4_mzIYBPa2*(*{!CI}&p1Zwo~Yuf00RR)wz%4?i8#G;aQ=dISS! zH=&JyS1}SvO8#-qZA@Wd%U4}ja6AsW(mw-HS-hV6=}LJ6Yi=*?-md%lIT9_)p;q5- zuHn@eZV*EC0|B8hhFuIcxYuiYF()+MR5peAX|mgM&XB4}E2=m60rDRb>%FN81%9Ge|ycxSJdf4#x*2^Xa_=3Z!rct94)SBrQ3-fO6Q^ZX!UI4PNai!Hc z!w*aP-pA10J$~gZlg+q4$G>%~zie0u_)SYtF!Yd=(cBJ|$O@bbd1X|WGJ4S{r-yB0 z=naOj6@{K_7~@Ea1tm0Y4zgCmFVI`jeJ1#H_BWmMJS58um@}rYj9Frx2c?A6>d=^I zs@B{!9_vl%mN%{r1KpjIcaYRSkY}Zr#gvV+wc=&ja8f`W39J6$Q7b#1JH73K=KG^K zsmb`oBn^EkLiyl7aVqKXQ$;d^dbSfgZJeR8GxxDlY+Bt(Uu#jCYy4oc2v%o2^b}ZQ zdWRn#79!}UkLq03TR9!=#)^V7qUyNaW!W@lJR(5)faP3mQk1WE*~wjfMyWu-sYftM z9TDFoxBkzgrciPsuj{x;KFGJIN&x`~3T1xK#UFeVyUNaoWn>>k`fv7lY`F~D9L`5I z)nz9|KY0Y6C@dM@H`_gTh!}#P78JZ1*c|QlsDpR0v}z~HMiCsN=F7h;uO*j+tMD>i zq$LKcY4JFSmYk_Y>k_nhT}!UK6V9rjm}T(UE%Z25deabiJW{agzhZBG4fu zkx0zV`G7J6BDA5D2Gm<-Y+|3sT#|{D^TJze=pD@t^%xG1b)1e3r@dL$!VWZ0OM*H& zJe)ka(=F4ewc9ClS@*2W&h9$6;k)dDymeQtpoj|p!ubCYD1B1Kp(6omW~KLgV5fia zV!Nq}H!;i2VPNOSPUn}->+WFUxBZ;p> z+2`1STj}`j>-y_syh1JG(w)U!Qw-^Jx!L&F>ZnvPQIp=4Oh^vg=ffeGnndBn8MBPir zb%Wtgk9#eMYZo%#*}mSRAc5`~xV1N4ZplkstdLxGGmyWyE)xR{TGzuU4nFE;1UumQ5^ObUjcUtJj}LkG^Z->!VAiFM9yf`Wsy+)d8< zZR96WuK3)F&yF@Kh;U*S|MX=yTx>Yu9n-!2G#%)YnXr01z6m&84#;r>x%K6Z?Oo41 z)E(|bqwro|9yt0BoNLQX@ZNpbCH`*9m7eLczW%n4Un9{w!Djv0#zegE>A5}oZl zZ^5PGW!Ty6e7(Nl-KNt}#md{m=FC7IHd;OJh93s$@gkU!o3)}kbpBAD*>n0LMO36#xlA}; z18jrYEN&e>lWj~j?}q~K*X}bai~^I>z9k22Azv3TzmX|JnA7_$DP@yo`%pXdLm0pW zohl0S5XJX+0z(jW%cYGTi}S5+ z>A9%W1L31-fjF6C{CfT6Wh-m4l3LzT$N9gD`>Ib+xUH*ED@i&@E%w+!92+Cqja|CiaFK?5IxxrbktznkQ6OYIB zXaN7%(_>kj#J6@|5d|(#gd7kQ@BL=gTl%ren>$)KW5J^d0|WCE&HL1AsC%vR(B$d; z>s1D5;Z--Q-cPd=SO*#d7JgUDb=2iHE?nJYd6^AyU|^bvSlH=4*t!p7PYlF=b+Hs^ zH+;dTq*RTU+E`qqSF!axI8GMhe`p=(1cObydU|jK3G@3?h83g;M8o289${noFQr3z1-_;l4;3A|lChmzq4-fjn`w!#gfk-Arv?@F z;?uQej5k>;#6yC(eC98+1u^dB9g|G8iku<3+7GB@)n(~70;|h~q zHr^+!T;6kml48dO@0_BpMg3j`4Q=qk*Rq%Gu6ONk{AUkO$i$0CR?L`~-)$GSXNNGD z7s*gzNz>BgH{Rb?%9d0k#uXEA4**f-3H_7 zOQg7(iiPs}ry9Jw`9$xJ7u`z~64Yuxs;a*My)2_)-I*0pGk0h~of*ik zGYP1$P)Ee}JpwUq**(%uoayT`jBiEYWavj`_@}6RNIfj4{B50>G!I-pl8EM3t#*j6 z@+f!WZDU8*?6YDWa-XsmHr6>(aH$LP@zET*E@Rw(Nlt5yHc-%jEmJU`Z*rPh_!+7i z^r{jiH=h5Jg&73muO0!P5Yaz)D#*mkH^VaCt(S5C;9KIVsym$P3p4d@b1$t8Hi&Dr z`{B`u;xhnJ?;2n#g-H*F-*4qCyx*L19aI+oA>uJvr2vq(vnt-VmmNqJYBC?s9cwFt zVvH~kXMJ`mIsk{{{||l%coVa@kstf3=R>x;9cJf4xxxmx4c3|Gl&x7dT}E9ra4bLu zkQ@`DN3WT_BU(f~zv6l(h%Lw1xt++sv==0+Rp;v#I^`;rBoY~EIba-F5eR0a{feQR zS5J685}ih*<6G12e7i#l^eV3u=zhM7N7?Z4zNL4nm(YDY`7n2Setq@xefLTZ=W`%f zmS6Wg+3?{M|9hB2=rFypans%UMTGQgW7j#3bL%4d-u*w^ln?X!=JP1PO8}srv4yK% z1B44&?u@YN%9$S5L;F~+5*lQ)@Jk^=Gm&``%}u=poukS665#IbW-ZXd&^o#%v=P)o zt${gdWF1D!xEDlqBZQH%rSqQ074uUSI-jowa)$Nt%gR=r%}>oJn@hQEBIG=i3VR@j~DmhXtqdhUMlb-qnekOX9C zi-HE@XQn{q9_Fbp#`^!O1)MrR*Fi@Luxj$-MfMX~tObiXNaUv7(Ap?UF$+9iRZ1iz znAdZ+dOn`8XZ>5-G)oi`uZduApb1&p%H4NDv!|v*^UGr^XX-(J*$)rj7n)Ee5BBjYozC3vRd)Y(~jCw ziJ2af>ItNXj*q10X>UiH?9V9Vd zE*!-EsC%Fo+3<1lw2`;>AM!(xB4~@WQneB$)mOffm>`Tf14eC!f+2$ZrlpAu+tx@2 z+jdvhX!2<81VriI6@$z@rFG#5hSv|0u?db{53)macY2mhU4<)kddZH~z=w!Rm!16< z`lkcrVx;1rQbbH7UH*-pUDIErmk5oqXWF6%b~ZPso6{^-kgY`EI4GF6MfP|a8KtmR z2&!B$cA(bV4FecBG81&aAwvy0v}=ByEHq#{=3QsY+A;U>w-(Wf9?0^0qpb+^U*IOI zBN7wEvICbGC+Nv>cE2JmS}2zJK+IhLE?k{DVi&i~>&M_Tet&+}TW6djM##ny8{+FD z);!Hd6t}j_Ne?47e3C$4Hv}uX)Z9i9owRi_jNY9d)<4Se<_86#J8%i4+!68iz|L*HiidnLT_zD2gMZk$9M9g#fk zC`G=B_dN-@h-D1H867e2t9Ii{SPqRt>+9q+7(IUs_aG9z?Ath2k=kEsSQ<^Vzdo9; zXsJHyh3*b2D0lN)a@G50bWig;Jpehc>O180r%Xh|`eMwVv0qt^;K#)EcN6N!awKR7s9RLMps4XcSnOWOOy26m;F8&mixL8OHB64Dw9;DkHOf9e4Q%nycWqLLsrOh`@gg20xfP zd=7)1-ME$D$Md_()KJQo_t}~SC+*0T-xt#TIh8r zk9L0#$S5UP95o|#OwE4U-`@}6I|GA2Y=WV0L_L)_&QF_{K&+V#lC+;RuXk&|a%`-1 z!HJZ)_Mvs;3q|DdU`iJFN=@zN?ROqo`}b@I(W5hSQ*T|mDF1=a&Lng*0yz{$N6GOP@_Wfb_ZTNnnC8uw60=g=YsgwC>6GyGU=4k*mNib?xei?(AGCd zm50sNk4-#>lq7mYVD^F~^=|n4i4;ycyy^+JfA^uYPlI^t2r1>=j1d8&^f#_s(sp)J zT*HMGqrakpHY~z6chJR+<%%{LM=e66a4wYXee1?rZfRA@Km~ssgpVE?xDdJ_`b{f@u94vc z{$bZF5dhgA92h}juGTCEz^#I=wnxe2^N}XCVtI2Me4OJ%&!&+RmljVE`K+~!b$^~Ie>k?JgqBQ<>}eBY^V$f_cAdP1 zf~56rol0WOpme5kP5>p#iT*$J^bC7z~z*0%wq8s_AWfhzk}l!CS# zW`ish{cJ3a8poc$z|C^1KL=ELAzI-G5hSu|r z8MVV*K5sB)=RSER@h+6 z-UcA)RYG*q7tUsl^h}X~;Wy&+eY(g=SaLGuZh!B86c|OM5=9O}h#FA9vT$*fFfiBV z6BfrNQ8d$>p_GvsXZt_b)n|q<-NUa?^M#+ZgrS*&m5XL_r7?X&uy~V}l4wrg{8+W; zv!*M7aa>x3uze#Hbl?Rt6Uo4>xSPMk6G}dQF@YEmMxX-x229TP{yw=#~GLh4tGIIO?U=o7o!DJVauKMQf$Rs!BHuTG7 z3}a>2*==s5&9|fo9yl2}PXz~6h3VIhTx8ZzU!xwnw57sfF$ zYyNu&pJfwTy`UBf!Irr(Ym$1~t%I>Nf|c=Sb)@tS(h5gcPWmZFZucuhaYN38Vtr6d ziYxLM0MZj`NCbr1X4I_TN|izLb~#l9tAm(m)N3`TXUZ1bl7le-`=t$;{iG0ji#47+ z9S$uj4A5fbgUO|5xPVFK>i=L|WKdm#hpP#b%NPuGCV*LG*8E~%& zjyCJ2gqpO{OYvRNtkN2tS#WfIm4lfaFvZb5eJ`ALZ}l+*3#sP z?|dcUYk>Z6kkk=kbVz`B-qeZXn`iqsYgL7w|DJS3^wsIugH;S^=q)EYu*bLiZXti6 za6Cs?(QjRtrvwJmEz5#6vfiv0iP^GQPsN)sfA`oRFgyRl9)-kP8y*SDk4P9;z{40= zDBCs@EARIX=l{+fszlZnE*B6TMiom92;rGbGXSMRL%ZdDk&8=E{0}YkIeHi33o}xv zgEc&*|0S@mg{wI-qJu|wA0VckgA=pACjyot+CW*X)K5Xq| zyE}yH7qBjPj@%hTM)u7FB9Ndm+MgrI|1EPI1l1AIwNFA9H|BUVZB9L{k5-6q@>-SI z`2}F@;ja^}Zx+KwDOiGm-j)H&yDSf^5id_G@K_UL zu?r=VjXVYg1UZ^;G90s9TnIF8U{<(+5GrNABZ?D6xX&d3(&{M?I6Gj(HRU8N z;P;>x3PYNF8oZ!}-?zHK+t&ej;+VL9*86>zbh0GAAEm)BjcS|X3o*M-dG&T&R%VFt ztkI$NEU0731j(PGL0Fhq!j3wsNF~``(rCu&EK33;eTv$3`d|f=z`4ubbli_S)n%Hn zsq}%<>pla^z8}!{K$ZJY49e`^;}4WTl)VL)p@4T)3SFRJZcsGOGS~z^RMWe+Z(Z~E z_athh|0P715>W~V@gG~G2GzVgKtVPKk3Zs$hS#*%2iTp0$?ZB1lQH7jx#N^7e3~!u*_5KgdBgGv#%=^l`kZ+V+s%P~gkAu~HOIhSJWz;oWS2a5LyRE{uBK8!T)!0a*Y*(=#2^-#{Mp7q9lV)&sbP#Va z*5Bd|Ot%8X1e262E=%b0^-tC4{SRI(-^=S=e$5-)0}E+Z2Lh}#uD^-s+hNHZtK7(+ z99Wc*AJdp-i1qgE#Mwiq3SI0gXV`;VU=Zel?DzCS4r^X)BdXyUBwcC}T(v_N;5H;gjJ;YlMq=yO+cy^Q))_kyMpWYtnu+ z)TVve%C0?zoq86lLIzI3GKHZJsNRtze_xF9fj~DFoW?+s&yY3BT5%J&6@tI%(*3r} zK_-@cyYOV|Oy371F_0o?a=yCsXt4bODd(@oZ`nH#lT+7h$Deq*$tP$-$S{08T&Bt9 z?4~inO1&j2zI+6uVvqnjnMeaIL&1tU8zSLSvt_E7}mLA!2H6Xw0soX z4MCA8^uTKSt=b6l!_YC+jZ(&h*Jo>6P6m#D-`kUh!jg;4i_xp{u$0`BD&1^r7I7rk z@NoEhIIrAG*Q0PYqkO}n8K^Rs1)ZY$JIBI{Uz+$H>@iFWL0c#yn73-^T$WG9ypRIa z8+sB;D#vF^4+dp|uLG(a+pWJU>?X)@zXx1*&DQ(IJ_}c(!B)(sQ^t&ZPTvVY zC2BStsf$$v9iCRQ;9dM$90v z@*AkV^~-}>+n)MsNx&%a$LK9QHeZD9QD&}%UmVeH)2M<}LZE%Ty{_V?^iF2fiuqsS z!b7?^SmbcVmfuieE2>6IcKo{cWcdgCTpSuQ+6_V^Vtjzba1h(Bo%w&5}wl&TpPS0rrP<_#$)h_ zP&!(WBo4OUg_d_aI`F44YUUkk`W~7!LR>u_QP~Fs2rJGv2LF|J>A0W+=aFl6eQ)rx zvul+v!m?N2*Of-SxCo=dZU~DgYqSjxrOodbD-~I=HmY;)yXjz=*Vc(_Dd!*r zP7rKAFYHL2bI@YmttezSt^*qvn*$8eZtQplMY9Ekq zR!H_c-8>4)nAZ^sr5HlF-yftm)K(EuXEIplNNpUk2cg8Z>lb3Z8U2h!Ylw(zIzsw2 z2CN4JmVhWsiGsxy9S@RgT~zqQgH%&%CkdVN`q@oRvcK70P!2NMz!4W^*oPjHA6HRL z9fZq7U~Akh5Rnw1mAZhWNkcx8&=A}|+e03j!)$Nd_3zI%bE<9zd71du>{?Veg+cvE z{*=@O!EC9+Q*5a%$Yu*7pv^=LcB+2+r=Q;$I916=3>k1Qhas4Q&tubTz6_V=y3 z=+JGGMrP=`xM~!eDZWM7jdbay2MjBaR7YUiHScllK|~RRfp=9X*+mh+8Z@nMRsL27 z6-61&dfsxg6ypmj79Q)iAgVjD$w1@$Fqwl$Pz5WLMc|r2>PKs2|6D@ZYagg7dWxmO ze2S*2%PE0nwWf3?h?)eh2HiaYNn0UUn%LBP(?8KVyPjnx^J{SDYtDbtlsNS3t=X-WmmEq>0mVc*HtTm zN5?l1S`~G z@e7)+5dA;ACQ7Jg4n&HauutH8;pBmv?FyD~#o-84%q{_TmWvI~TUy0#Y2CD`%{b%`#g0xZMRQ z02%IOOG3Aa01g5ARr%aBaZVBBwm{3j-#}PNLXT?4rJS(!wIYm+{fMho_{aOzA3|jG~Cqyu{XMj|9j5}TTXOc{ z_=wv1*35ni5roKTKN-AbmiNS(ZP=|PJ6xyH@{8sRt7EiM1G83Fsc)T`VUtSO%r>NH z9~M({bXQ!eZcOUXllE2F$WT9y2(lP=7iR%@R`}GOEm`fs*>=JxEv2CCtUtMObvJeJ z3_XsSbidgYIn#(ec@*66a88pVGSVRjV+dlZGD8qs7SoQST=q&SvpjAf+^~;^4wUKq zjIqg5phOHVaI{1?^ROSPN=2U@CzCdnh`%X0)LA4 zb7O9$v1#Rgb7I=_wr(GN2(vwgTuxNPfNe(W7TdGcf@}3557u7BXNhful(&dXc+xT{ zjh?46YQO|9`HR@sd8T^MLZ@`lBC(ssBwgR53D;3CJ>%TT zPe}E%40D{RmuwK<5ez2;scmvbhM~A;8xThHBO$t-Ipi&VZs$PLDJLB@sym5JzOZr4 zc6{2(M7h&$8EvQCcF9A~516!X?@bURj-rsbC+r{n8-c}4Rp!fLkq1gc#Sb$wl&Z`z z2C2ezC9#m?SM%O@JiTEhH|3%uQ?MiV#F$e`bqQ=KsqnTv(`_B((S-i;Ll{C^mE7}b zt0C(NZiX^uurGeah=Y`jJsM(=!km^&!dFAvPN$TRmN?IFM9PT_uF?L?&*i)qc3f*< ze;u3gcnV@&-8W6HCNl=zRf_)oPy8rlw#Zu$5st8)|q5YoH99lUb z@uJ}d5KELGtE&hYT7^L~2P|SK>Z#cQL|0^)6uERnEE2_kUg3w^D5bRC;wtn&7ua4T z*T0Rt>A5H20l;~zquoRK*|;cyQGo>ff|>}szHOxY|^0!BJc8xHmH+e z>6axKM|2c_iEHKvJcl-9z>ta6T4tc=N{bcEs_$#$2yB(7L>>DjIhDc$><(B6VeTHS z-@?!E%-YMF(LMy{bDzp-bNKd-K(Hg-xFSBkzvBQWSVKoPKC=EKwIkOqM3O+Du^<8h z`GNSLW`Yij_Opk`a=NH$kxwn7 zCqXhO1P8-))BaR2?ib*1-M!>KIl5dRR&vk@L)vHxK2v^38&iKQW%H_QO$r5<;wxspBtQF;bl@Ga;Q`M)Q9>oS4RA#ZK&&o9)yCRDFFABtLu=87B_5 zJ(&9Agg7Ic%E=odX860xZy&?J3BZy@|Lwx=1cNdhy)vRr`U5CT@Pif~QbRHjBma`U zqW%$Rq=43P*AjQwQ!x9&Cs0O=NFex4JQ=rKRi98#lt5wdvTAhUz~KHbvpgL7BSsz{ z)+0kJFZd`C8SF@plI$1cCz*hlc^3AVT6w5Rw?3aW3#t=2uT)*kUv4LDUvl?AQ`^yz zZQC)8(9wU8_nM6G0#A6lVl*nsn1|(+GvfmIArVQT_Hq%FH<1ej|4)>oodY3UkF%7C z&?tN;7oxWZ%`D$5drYk^>VZ#$X!~sYDp>EFM3CbnTOGSS27q%eUscQfo!2I3O5}kQe71`OE5<$(=5!+D+&H8>g z*CX{;vH>BfqR1P-{tzDA69tj=ulE7e-2<`tI8Xm&6qc}ISf@WSYuoa4M z^^s;E*ap?1;|Ow?QZ5EX7U)jc9>zbL43J&UEU4By-#XxjB9wlSocdLb{wW4uD;O*9 zY64k2;~i}SPYV#0%%*>thGN4las@APhfm%IVtO~UHK;ZaeF+%(ICB!>Cu2In?j#mZ ztj_~`KYl~tfMCb6jcBo>Wxa@sBn+6jnIq^SQPZo%lKtTzY{$7V2|^r&dv68>%{+WL zE+ROH85iUB!yO_hgM|a9$Wlm^S^Y0#2%PP_l23EMYqEl7R0JiVwc2yV@L?y2^X*x> z9*xqOYFBM(4qXE{!lk>QNO%Xr68vwnSt3+vLPm_#X#$~SlPiDi#P>d>F$HUZN_xsY zh*HnsBK9%U5II4W;dVfKglQ{IB_eYJA{bU?mP|5B5W#b==pHQ75iuhACH2Osj4aK% z*%D&F)|rbG&Hm#Ff?#dhjPdhnb9T7hUe$yM$i<0D7m0=LWs;_R${Arzd9jxncbnm&Pnrn8MgS4*J_c_)oSIqrLLr8f5W|O z`~;zi_eeaR;LjMQCnM6vF6^+R({*UyH;-1%4z*a$Abt`)QP8-}9MM>S?JUHpl`#5K z#BUO_fgox&hGD%h(IpdpW`cxa#?gl=m=#?cIjNU~VFw%yMQ7{i%Vym-G}HSp%n_9+ z!t|}ecr+C8UU~__Y^nRu-%wG4L-8?^@F)2?ya;Gvv`Hq3KXtE@W$#I7W_}Jc5O=)k z=a565=Me^-kybd1*@B-Xf(2kC0jkTn$w7kP5alKW5%#39wP*Jq9@_?j2s-{`bkTH5 z$e>aCUb}D>vcOo;={N)wYI#Kya}4+NuxUykRLWu49-|ME9b~BglW4)|<_Oo%LByQN zi`Rk&7`d^JJPL8QA)()gGi&bQUs?Y#4wZOy+`R}Bz%AG?aCJ(^3gyqQ$tjUn>=)J) z_4>+|1n&0+KFy<22^N4i=R3j~GtcpjittyPLOe^0vyv4L!=eQI$m(ySCm}DhqnjhY zces&=dRl#4NHi^1o&9KUhE->jdnGA>fCJN^r;taOzr18PWEIfjcX&wz^}$BL;Ts>8 z8h$~83ig4D6r}MXq5?AxpU5#*c|UJy<2bM(F^^E1gWe!-x20zj-+dLEsU0&AD+HHl zN=8+3s4hjK?P(Xj`zU(A_cbRt=A(euu!~AY<(r?EZ- z3O8&g#yc<+8fx?y&jFLpMVzif4Q+f4Uj58Aytxg*p1eqJJ5-?aDVkY?bf=)2vh4Dt z`AqN_TXPI2E>NIQ@zYg(q{!8T8GGPsUW`}x|3lMNa7Ed6T^b4Ll#vEO8bm;WArz#f zyBnmt89ImVkdT%x>FzE8mF{NfocQK>*ZO|I%-n0%ea_j}-sjpoqs04y-_RVnFEzGd zy>+iomc(+vl^Y0{o|TYMPE4w>z+%F3woSvh_GAz-J5S74|KS^#+Tz;SRQ!gi`*V4B z%ERYN!s~ZbS=kAgZrpBRlBOkcOuR%~nWuCX8^vTc;s05Xv;HnlyePod6u>+~1l4D- z*r{Oux97xbac6+_1i+@ZOm{2@7)x`J-*>4n``+Tvo5&X^1O+Ef1{MVYb!&>X6|TI$ z5CouDx6`lkV*il)T#MSqKTJF_JuzWU<#@~p6+IfvEUpPx1x`_%x}}Y~xRe4IjBJ_m|vLsljjMpXQ21DKjvefHq!9+N|3>v=`vnSiaKUf99n< z`q1G#BC&g7K)sXnXh84&`eI|Cbvo&&ChVkj@w(HC`tnutx3 zcadCf+LAcOyoLjhmX2SE@6ZOwC#svE$d9o|wEClL^S9(iBbg6QG$fK4*9XsF{@{;F z0ap0p9*l1JzBC`7{J_T+yHuEJhL)>OxdX^raz|kl6u+f%`;M!7fL6bR`BedZjskD9 z$yvt;1)$4p>Jkq|m&{nwFP`W;lR%*ourE&~vC&uKPQ#JJV0rnJvb1D5?(eYU{7h{= z9+Vt97ZiOYq$DRzi?($|WRiY+%i#lylZjCwWXKFTJ?A}{0S%SIplj3DL zQQ*hn22GQjX7&dDWD@c12j+D;GUcBr{V@eG_-`695);S%ydu$_N+kr*ON%jzYd5`6 zcCWLr{K-7*cKV4}?Z}6%#5^LXW+b{ZkI{J{Bj-uAm6`puP{VdGbKVyD1wmKy0H@!P zrK^pRb1VMAH7JBQR&)PB{7lI4B(-)rE>UF+H;og*t0&u;W~YeDrqmJdKWWzSB=d@I%2cLQqku~eEDb|!jXOg`#}`eRzR z1-Xa15VX1qKRv#SWXC4WxiYWg3U{&WO_FxhDMHDYxooS-szZ`YZvnCdzG8ordv#oe zHPDV?!>u{zt!7aF`G1I4*h6wnlf+W|ToIw%!-)efw0Xc6&gqI%D43HX(KiN)6BG0r zdCmr383nASA`QUlqN1+!dA8ZZza@w`M62tGId(vkeT{G(Lt|8c`OJvl)ow zb_r7}yi2KH4FVot^9%hwGsAYN5nmp|T7G_MyN| zaF_}G7iY15XGt7@am)3_Zql^YF zeWW@Qpn{nPC1quH0X_9Iwjx%>`m1e-9Q{J;6crSSJSD&E+Z}|~fzgqPIXbNwxH@8Z zGCiT+KS9SZQL2s6sk-t^a3&YD`yb?K^&Z0T-p6-i)|$|-9`BFYD^jAA;{QR?Tl8-^ zj^>joieN$E=s5ksBcm^hZqEnA0@mk(B*v4KT}Fr`bV|FU=G0QGQ-fmM!6_&V8NVF! zQrv9yFfv~aO%9n-^X?|nOb^j>qW{`YWK@d%!#}+7YXswjNj^ooa-2U$#WVVdTosnC z_&-?RCvB!xuHqnwmisMV)xr2DMX{bOgUAgtt)sL6C<>}%wE!ld1vFc1M7sR*6%8cy zr(dli5K5wIC_uE;f;Z_RI9)S3*+_$rIF@p2O}=6E45gaaIJwxm7Tatm6+!F384(}S zI{EXrRW^Q1g#06)xf!DR_t}>rRo}%(1j>jx&v?pI)15?!cbNBQBpleE%)+91fesd( zj>GSzDwg~fbiKVb@D7S7C`CJ1q=V#VfAbKk!!NF)lX^5`d08p~Rm%Uq7{j|*f`m)W7ZH>yZOVi z=+6nGID09_q{=MqzLK8a*FvLD8J29{>+&%L>2`D*8KHazQkqOul3QQ4>;xhOnRl^E z0fvF31VIvG!$)4uuKByW+lU$su~fRMObx9pw*7Cf4XxTPFVTo_MawwNHi+Xuj0cW2 z#~1!62fyuHp{!S*n4RaIUL0sW_cbTK%kpOo_K5q}8$@EF3TK6|4k59qg$Zx=f2-wD zvjzf0KeNgoUr^c2$OrKq%k;n08qcMBFp%)WpWUa!oLDPRScw&Q@IYyGV?imHn%x>l zDVotRn?yk?%#wU3q9W~P?TY#nM1I3S@Q0t{P?EZ+dFY6H(`YM}wL6u>^U_(7|G=N` zAOw|`I!qg9HXw-4;*wAy8S{snB9@dMGn`N2HZ{c6c^%iJy@^;KHLFn7n^$($a|7!X z7B780A@RT`j|DywYWn48AtbE8&9-nbmE;z$)0Sq8)n|ZjV?-K_Fd;Le7p6J88ny|?(}agvesyNAMhb8G!+h~pW+*$SOmv(eVSbcCj-Z@9?>?JvF>-E0ZU81Z z7p?Ohu3yyrn<=OtuQ3<#qQ?8wFn31#xlsxqk*`?hl5mksk@41V$38tponc{D74$aO zaxiWnO2!4MxJZNr(ObecOQtT;N(c(m|8oJHZ0OA)LyzqrHsvUT-FLk(n;1hpP5T=% z@3uL!(s+UjCNm}&Ey*G#rjGzw>RO5XARP2o{+7;do`rtyH-%N>o8pt1X-8 z(|Tz<5qU$-(x@+w-@LEf@2ocG`f}5wBAwP`4C?SX8(^=@zg#h^Ur3+H7(`4AMe^xn zM1@lTNAt#i>yp&78k(~u?CfuzT^ayobbYpS;qnZ=-LZ)VohKUo#Gf*Pzf|S1mg_L?HHj%#-5qVr*cYm`(at^V#bNKcX7z1|2*XR>}(Dj zOQY)%TQ@${KYwo=F{*urG&S8fj!H-}3InvtTdC+-Rz zMA~l6h-><}+t{F#hbnM)1a2i?cZtvRb$xVR6|!309b2$ETWOryaY4smR6?$KeJw*D z#Qtg4pIM|AB=&O%aL_O9=0@dFoyu2R01kFmNa8z)&wZADiW8TQ3cg_D!oo0w*SI@7 zO>K7KgVk{sYQpw2Fa!D%L$CQ!yZ|Ztq!KxQ$&Te3EI9vgp@UUTz<1w#{ar1^GtxsO zvH@TpBKzMkFPlda8!3{=wdLj>!kjm#LCUID z{zyHFHF7{fKTT?)(SXu=q7T5pe2t&afBUj%)ns}|N+g5T?KZ~v`p9;Q(!pZWmS2#E z5M{WU3IyLv65%cbn2$CYAeSiL6$}9d5>n;{(?xVsJ;T!%LiuGTZqeF4YG|1Bb93*{ ztHme=7XVb$vL`DWsFW;iy|6r(l4)W{jkwbG=jY>D-2}7VON7nsC7PL%?3s|6Q0Wgd zj@f@_FaYnPp=RxG|KOiunP4%cjv8Oh9cu0GB%tCqlsw$Sl6GOER{uoMv6Wg9K*4LB zId37;5pIMl*P8d%wb*%XC-_Yb#-k@*c+Fc_eWacpSFXR(n)D38`TbF+wWhDXnPIb?zPtnt zDO+vz^6J%zdKl$V=b-e5bnG5PTr&~hvfg3h!P%knRDtOM#-d;T;if@D`t#veE^N?~whcmp zJLx`~_sY5N0*0yaaB*V+@moL`T*$EnkC_H(QrSB&WApA`kNEW$q=`$X>)Fc|<-89k z1NTOMJM1_l=;u=(@(ZOFKbWqF>zbI8tcr^Jsn@#kmO&G2>Y0*?dQDC9nt2YNR?j*QG|**ESa6N8VN{r9`273xg&*G9Lf4)5 zA4H7UDj$=J6dk$aAAI9EHKs>FDV!MFvaSC6G;LTxzut#mM0WV*@0X zbPOnMX=CGTQa}jz_|;Nl-0`$?Zxl{BWu>rJ{^|;=lIx97ZLiI{g?#?3hRvZMGhg&$0Ux(%_2E{LNqcHhL1Ig^zESDbfP3sZ=66=T%`& zzBbwA-`rgPLyiX#Hf!AmSRu&#g*a}~A(-dlLQ;7Q+fmUD@^= zDujG{b%dejslmo_)G9^k8ljAxg-S1etA(?0nt+uh+!@VHRI+ z!k8jBdyxW#1ep2^jYB4yO5hRA?#Dg|eR;@-K*Ta3NU+`mB}9sU<%U=oyBc@!dA zJd*=I3683PS&M91SA%~53j>Ye@Z;|$zNo#))Srb5CL}5AEZ2Wv1UX}wj5a;G2RTK& zHknA07s(?bv_9-n;i9PW;Y-JVu((3a)e4%oEC`!q$Hu6Ql(%5WGWv!-H*dgBP1zQI zxk}njFj#B;wqu%-oBq;QbSIx-2U;;x#Fb>wo0K;1nU^~>9x!>b<;ky)6~bRc&aCAa zHbK#wlwMA$o;tvPaf?lC1XhVQE%3G!>9JEE3o5vYApCgY+>Xx{&3 zkawmc=RatHum_}`aLO4uIVTY40P5NGF`h;PMUB$?(0(x{@S+o=WfIAEd3A73H-6TR z%qfGJKX-3`8~P{bEUteAH1qF%EzMnvPrN*^>&@2#YMjo#*JFbKK6{UeBJr;B9Ew-@P2T7}s20$CX_NB2L$;bGmWX-dp!2 zyG`r6oE`@MX9fg!yK2(SF!oVbS05OG^!Q%ipX2D~J!DS_V9qsGnK_I6Twk`2TbkDKe& zy7%MbRZrt@hoSm%9o0`*hLRTAN={< zR$$to$->7)j(RVvX8KHj8FL2}vHltuO8b0T)Z%HcPxo{-9cMj|mj&cUPS;-Kq8{%)ohLU& zx%>D>@#z!Z3SM$B>v5_xbVcpM zb{jG`xAfUtXQ-=%Vu5_0aLug8o?$NHL!#^J1n=)v_vSs^XB{N{EZ$=8wdFrx2?o1; zme6NJ%6!;{xlFO`!m|4lx+KWv@mJ)#7Mck_chWLL1WXdWfoU&N&dn)uiq;OO=q$+? z12?=n7;Or~Siub9A{8g{kYld0>_f>a%jDmiKlx;380}^e{3IkCBU^#Pg4Zs&NH_uI z_8{%@=(D#*?l!h*Wc7|mqn!a9PuNc9_0@%$jCl^ z@fu<^!3or{6qWW{Tbs)JsHz2t7`kk@OS0)HMt|df;}omb{Xh@koSUcOeUhYFdznrp zF`iT2srU^kDshH57wT2 zhMy*(_NKe&#qTT0;yaTQ?TmIt_oZaiJ1b%9+)}O$iSU1&m}5Z7pIMhGaB}6Ji8K}R zl;w7Z=DJ8a<*hm?l?rL*SRcml|8oIoUZWj6HO!!JT)fb^REK7%!m`TOqs$*ye0RJc z=a3Vmo!|8=y$i;~jle#)fp6ep1iAL=JFmV5q>+bUCD5Sjs;fOkR$iXNvF9X%HDyzh z-#Ha=t-Kzdwfpm=2WUGFUJ1OoZfF>vqUruwDTZ6j2ip1@o52;Co&)pyB`tvSE~~2b zcwe<>xWTbwKVrqr)%De}a;~uF?7?*($IctYj_qs+(v~D#D~5}vCjuzBGZLO*AmBI6 zctvbe8FW54wr~W=B6>K8Hje}ld?Xu=&+&g;+~DE_xFAh4U!HW^*bD2bs(Qe?5VLm0i$->u-^+&XSNap7}z3VXrlX&k#b+!Ne^Olt8Gut8g~P4K!@UH=~SGhPdG zi1P^t_z~T~5v|c9%5=*J^{+Y36YuIzdjBX)L;7{j0Eq*;`1)!i{=pp*F&6~8{EW3XZIJ`$*0Yh=F0knQUM;m|BXR_0j zI-pze#cm@JF<@iVN9DjD!BL5;n@-VHS>25hbrdC4*neKV`Biq1y(abN-&eDD;16<72=aYu1z-Z(5Z*y&PF6C4VA-(tFB< z^41#FDj#)r|4)btPb#i%ohE86r#I=;?#s=ztQ{or;`Sl zh=Y!>?7EymdF+kfq%X5oV*t}j!qYix^>y(P!~aGfa10aFhJU07^Dn!E*G$|cxHc3E zTbEIay{6{RXXUkM`!}?xmd<)@_i^^K>ZKX^XqKGSr^x2g)R3(e(Kzmzr-y=E**Yev#XEFWIYBO3TMnBpg_iDAqZ%ocg00D4r6=xTaQt{3t6Qqnjvf7I6E(*HTw5|%73mZUGvOSAJw zO-e7&&*P8NH;ywiVP8btit%33-OyBb?R^g*ax! z5Ps9TerMJ%pL0Fnc+)P~&jj72n)UliQ-$F_51A2nid~zT9n~0?h@TEsL{MAZ64WR^ zHC+~Mp6d5D2Q;S~jK+d;e^|m}6y)%|@r#4I74!{f=~&BLzw5fBkX-x&M&=V9Z}ke{ zcvpUUJLl3sxe1ffKN*f}U1m!|*NiAnR=~fLoKj!ymU~1SE|CB832Xsbvb4@VhgW^d zI%Dj>!%_XjsmkUqVxUnObJAU~WxuRQ@>z`S)^S(I(6R15W1Crwqr43F@9Q;$937j> ztC|Uk)+k;4ogodrkY9L*BC#bHobQ#{7ot9szj-WIA8BNp5Ih^@^>EW63FUBetN=Nt z)z?fB1URMnthpMO8+37bBkn=HejCs0#<0bVGf{Fv&;t|b0lVjUB=F(cIdsI!|L!1Q z&MhB7%ma-|fX&f8tyePLcOag{5YoopnB9*jJ*O=j&IjZV%D~3~(A@$MLY(0Ln>7FC zw|1P-c;Mwf5DPj7nmy8V(ZWYN&V3aSE4}lVO@uzv zLLgX=_a29l?}N}|2oE&540zp^e`y?mJbTN6X5cE_H~G*RNbB?2M33kL10k$x2FBsF z>e538yWhBH0^Y1m^d&$uflt;5=t020d|2+rL!5Dk@7X0XQYb3{F{m7HypZp4D|KiH z@g2F!IS_>~)$A@#&;Z~yb(OKJkih$s55{rIk4-cFb9Au5nTJyzXbk8s;o+_aQR5Xb zr46Tp1VG}vM0lRe^KaPmpQx1so*L=k7qZB_iJaPyer(Xij`8)eF)WD=dcNGG>;~U9 zh9_H#p7_l8tmA>sG;9B~Vzgcw7d%!H`W#|+?=OS)>=1j1=Hm1b(Ys4y=mxCITI_zH z#>mmI%M%$!V!7&B1i=lu5H~&*N6oIJPz)E_N?zSl!B7=sDrL1JR}@<@(+D6FdL2W= zT{%tQ9A{BEo z>Pl1h)kpREuq(B(e*0x>)bE}oROD|1Jqax};zJD{LXA$OsfVb1i)+};-*Mnp7Z0sO zs`Q<-DeEn;<y^yL*qieo-gj>B@j8^fTf$2;uJWB5i#ubGCJ+iIB_NvC3|4F2#< z?vHLaGy5NrQ%WhKpOycj3<=Wlz3&j8y~fNL;(V`bkN3;PHuaXbzwR&Z3>a*Q$uuIq zRQUN<#KebTTy6#?;|rP2Z!1Ty{tB{m4#%(;sbOKI*YL0Ug$G%iuOyQ}|fdfY?$~ebFJP z>(k(OE%a#iC26zz6o>G}lpUf_G3hVqO*N(eXu4P*@{w*M05hQO*HtTf#5!a9Rz)@Odlqt`#EIH|v8@lg5nS^j3Qex5teN7}vJ_ozHRC_khs z!+$IHJRQtH+h$<;pnE#Q;L1=S((bLPNAHg~d{Yea7)|#K$IQ4|&p^z_^~4>U5`qS= zDE$pZF0S&)eJ0Imm#5h_ZfkokkH_G<4&#gZ!NA8d@sneR@vGXN>(Y(DzdY~*OvDyt z0F*NDawG6OG7zJ5ho$FY;mlz??j!uf3Go2{S7$i~N?(N{?mqgAD|>INZ$Mz{h?oxaYX^Pe@ReLf`go&31r za$gHLe;9ltev3-zJ?pgoNCTT1jH3xaco}vHw)y&9!U?_UIW9%=Knx;HnQtxlUVpO%`<~MwTvY{`nO%{SL z%e5pnzTbVD^jxB^c$njp3*rc*R;aeexM5B`_Sd zWb|J2XbzCrHLNQT=s3bHb zF3WMQo5w-|E_RK|+*J$i-Flko{OO{ZSRF5+Nvide^Nn<~)^bT+t9$*25Q1=Syxz*5e&Ts zw9e$Oc_d=kGi=5CnP2vsjH|zD5`G@op-93Vs!5g;5nYk=>rXK)oaFccIsOn~W`I>N zW)AaeIrv|-mjj_A55^$G8szC50xdQ6ADb~ncrWAxo^;)>E+B3_YW;50I(<_?w^rM0 zkiY}F&VzfNUksK(S7)FYj{ebC3(Ef6 z>Uq~tO89mxh(ZY7T}zX{*?C$51kH$noOc+N+g*}IfLmkvSGM_Ml)6SvqFa~Bfwz^$ ze+l7BPHTTJSJv;AHehQVw0CupJ&0$Z#7z4JjQ0M|8>22spF?T7yC7`Q3qnMaa_a|g z;B$UzK2P6H{zYo#gX?W~8KO)#{gR{h;q&(g+V4C&cdz^r0NJYjhD*fd@k@YIU1LnZA2f8E5@(PD(+@)^5WtocGgSIoJdQ0 z_a^HDXOdorVlnq?6zwo8wUcS1ND&cJjao7#%=ac_M_N%jjUH^81BLjv0o*jPF=>l0 za}8XgwFc|;i%T`vX4ip#pVk#rWptf9@@PEiO@-9no@1CmikUP%_VHi?CU`JAMA<>j0P%Av^x^; z)@3iSe!o%+S-=guR52`Osbt0$sY?+UwJL4)jNy(Mq+?=K0ZopM% z_3Hn*fd5r;DHcWKb(MS;C6-$}_nFds1lX7bi@}3&257;l&}%v6<@7~0+QC&vZobi} zzmyvjm=i&(o3m8q%35*jxo5#T$8yKL)7xwVA{MTACMGq#dL4)Lb6`!A;9h=F0+VK` zy#l|id;EtQ{#V8U2KllMg*pR)N@ITj6t$n z1Ua3joe=F_ov`5yYUG5A47eEyTor?K9cY7Y_k_Q#Ua#L@^>i=8PvZPwJ;wLxLOqw5 zbdUAQ-u>z|@U{=G_bVXC8xRy5v7-z%@$EYhgJ5bqsRv#Qfp#i;p!vub9qGvYcszsX zN7^DsiXJoqA+>=gLjFTnbcvt|{q?A$hqaD%k_3Y;6`uztI>EWSq)27yz_7Q;_JgKkKPhnZ+$RDmKc zGh+8T0h2xACx7XljzC*HVu$FoolrZcT^I9ijEU5ZyI&o=v>oo<;1!#ucH`k>(14Df zPtl@ujGyH~0Xuh-(A176&(O%1rfkJY2M^=rw?|sAptsW^`&nd>*w^2_Sqif!g?(#i z&ibbrDsHXJJ51rIFyvjJ46(?%w#F+8VYcHbO8r$N0Rmc(9s5DZoVp+sD+Cq>f5n5_ z8|C?kCUfg;dhGLr`*tuLr=g@V)%Kqo*YmHcTBY6;W!$$TF?2cpG51k# zGI;2Xe22BpT^Ma6HYUd><>k)V`g1X?JP+=ygKP#r%IQ=_EB^669$>~sK$DnIwM^zu z>t5D=aN~ehXTQX+Ad$Sm^5Gv2IxG+SLXoG*91%04=a+w}LJ6T8!jypc<{ zM!-Q8bJvN6L0W-9GINV>bWlQK5w5BKzs^^37Vm;->eW7IM#MPY9$&ZfjUKPz(E7Q% z?(A2Kh>NGc0(i#SOoY$T22XrP%Ox?jTBY$})Al7Un&PW!jvYpI>Uf?Vxv#b-)FvSO zBWU&8XoUcvF;(w@sjX|_x;=U&r|~y^hiI&oyC@IUPL$s$=p7p&VB3Pb3-YopJ)hR6 zUy14-j+dt0bDTDz5VG94uQ?L{$D_NNC4?gJXbR%ouV)Q*3Xklmu1t7da{=#b^WX&T zMs^<@=Mx65lIB5EtvfG@jBb$lZZ8FRyRdN?zu|WYf>%5|TZ=&kYIk|MAFONFH4TC} z9=VJWd!WZ2c+O1VbR_Jy2S!MHIV#(;bKcUkH`OE1bHqeN(HK^J(6ZFHx=1OJY!O+$$P2p(|T91b;pQ1~OT?*8y% zpSQxa#Dg0-LegIe`q@^SIL|1W`2Wc-AP<~xF)s(ba!D(3o3KEX@Sy8Wab~)oq@nI< zL?&(LwX43cD^$f_s$O*M)Zs++iYBQ);SkYLOaTp!+DUq^axf>Ta`eQUq;wnY=ozpa zugP6Iq74Vk#cXTEHvPk^Hsd%`F45DeHzzjL1I#hXvwe&W5>8>$7}X*6f}^xUFcBjYTnXDCN0)pGYtP+dwh+~hw!Ei z-kJfE!=|pXqE~;+a56;|erwD;|Af{O*9nEKfDa^E(C8`1ExU z<{)?4-x}Sl;f~XIO7nM%9~@;GsPYUM>^`3I+m;FVabx)qiQnTi05UkiFOICxC+UTf zPOyKKweC^glxq%4{{8rg?~hA#-R08o8T|d{SE}dF(w=ZQYp%1~a2H}j@eJI= zQRYSLr6oL!Ccwg!pB9t@uNG#?k*7ViqJm{u^Gb+Ih6z}U@3)!Ezpt#dR@;eseifpC z(+=O?ZxcP4!YwPcEeu9fFiDkr3(Co=^9Ya2J1_A>pwpYfcj<|?I|Ue!~&fCF@Y z+oeXuj%Xl#5Z4Sb6z929!6jO}?FrvzUQnTb?y#l$NRjnfB0?id3JH47gi;+v!!PM<5emSwV0?h?oyv+PC}js8Xbb-gUzQ4WKB*M)XOr-jCxNTAYT zjaUp>uBkyzM?3llLo?C2qAPlX&F`y-==>gA%{6+Zt<2m-8@k2@)v@x@e75E3*-|Hx zg-u*y$^WuAOzOs}T5#~apR|aJd*OW#3~d#QLVOWeyMr=95ry?lyyY!2XVFx7Xe?I4(JV-kM{vysp zi^|s54ouq?2&`;*U8 zr09KHWY5aCW$$tIyh|^58R$v~F^#wzl|5R?-UEighlSu)zT+IAy^PvjS$H8L#Q35U z$+;MbSG2t%^!vk9_df5slhS`ORy?qe(0?`*c;eQ5h}-$-weG!a=#-a>3_39KyI*ta z@kO%D@EhYpj3;CPsV4-il5Si-LjuogPmKL;k$m*ce*>~H@7k2^E(GX)Tg&4!rhE_d zg`F6=c_jVE!D`oXJoS}{pJjm{5YW*gaKi69*UlQmADh59UGyTDt`6X{x|0v>Pw*x7 zj0Zl1-g9On+Gd_YkOw)*iM}#1*k90{01&YwzNH=rIdl40G@aoR`!c|Z2OPeB_U6jD zG-=@*)=RwdbBJaJT*GU+L;<{OqQ7;VX)8i)QEr=%AXw7pn04u zT3vp}3@A1%Z{Q9Pq$@U0O8Qw?+0K6?XW?6*=A+`k4P#2mrPrk3@WZs)bw)F>m-(ir zx^@#FQTSn9fx&aE{gd7l=?Z1p+k!Qwm`mNHqD z#nD|%6&|gXGsMn*(;xS2Ttvw(Grgly<8B<}{Yc*%0UlVsxwb}rTXeGGPt*?4h)_2! zshEi!I6G^AeZ_pL88w^i4@!(aYf$K2>>scH%dJ0ai6eY^=#fB}@R`@ShIXUp=EB%LVW))W@o&OIIi+FOz=Iv)Yy);{j7SseIg0yFF9Pmn%>-OX)3!r5 z|6?+|pVxZscYsi*H}?T!JkZH9Sk}yAD-iw$(thGaiwG-QN6IYEFt45y{hlKZ-q#S{ z<&WKuc8Etx|G5q1;Cn2Z30%iBLbPiG?=F?weL-%>;po58;xCAhU8`MJ>x5<)L5b;L zvb0Df?+-gM9ss5MS^V6+chY@&-E-&!y;O$&rbHrlW5m<4_zgPnf?YXaKg|j8O!rW; z;kN=YTk&43G=fhfbBvlXKrkmnD93`>E(4*@c1nKW!HMz0SzQ3~_X2N?kxvCP7k_X; z3?Lp7jGs?>o)QWD|6Kme>id5#AkC*X9TJMyP)nI@{%*e7uS(w|2Q;GcIf zr~b((WN*-1*B+9Z!_^TmnAoc>wbf*=Q=`PiYR5Tb?|H5`{93SHsa0unjj5SS9O_Bm zHD1^j)#VhC5_@N;71J{QJ)37@!24%HJ zkhE&3JW4qoU61+lv5q&aKDlUyi!v>xQ>%Wzdw^$AE1M%YQ#q1+t{q_M|3)@V`ds7s z?a*?HCO2MxnsSH5kb&S!ZCskS?~2O$DlaXJl(9sNzK55@a7m-}$=+TE(#9V@O8#c# z=l&p|g+*}M_T5iCCt=Aq+8Z7G3s+`lAQ!(j)7I}?{O)q}L#ia3VU<|FT&Dmn@kO?M z{9&CDo4MYvY@FCH)@2A4>z%pf&1&6rIv>ZVFWJf-rV&SWi6Lrm`)v}(YX)Hl0P^hT z)s3^W^dyBDiSQn#0i6urGlI4k!~;aGN`>G{ErA2tu<4w@XI$ehg!dp5?JXi;M5yyg zuqOt#V;nF9?DBO6jlA~q|APmg1zlc(w)ql#PC1O>^E`+i1Vc~$rpte9iZMJE0UH7$ z)V04}KnY=^3D055kF%7}La)x9bHv3C@Sz9J1A^IgA0wcl_nil%hzZ~|AJ5ZHN)CKp zA<*-2CRp@nj81L+d0o5f%5vjL$+&IOMZ9J`-hW{k^jO=uU80Q(d~TzI7hsDYE(hLP zZosDyS828070-LI#s-LN9{6*2y2~x@aSkD+&nC8i?_yQ{Q(^eOon>evN6I=f-)_eH z?z(op)PNfC+ygNN&3FY&_;_@K^{sFGi=M;Kj$0*XWU!-&b^-zD|)&7v-wqu@ecH?6{uVR{U)b4GZ(?JMy=k&df68a`;Y zRIi0zdJun$)1)jWjm%`~+BAHsRt!Fk(;U??q~lz%h+1mJT6t3bThn0^S=gF_4}Se$ z)xA)84Ia)d>0kKH@Qo!#4IHWGuOexAjV8$LS9zdgsOV9+zOcOEepmJt?5AXKtC~P% zdZnTx`xf?*ypg$doDi;0Rpckz+tvN1RL}swrs2ueIy-TU*=^#GSYB;-I+lYQEQX%6 z-5pwu&ul=cIr{yHlu^(1LM!B+yhOdxblngCLNq}mfGTAn%8Op|+BbIVGE*(jNHJiH zk5j9od2^JCnwwxn#czO>F>Hs}W#pp_Fa*OKj4H-I;>WC2p9|J5`2@Z;X(Kc>z)q*m z`$BYL@rI)g$0^#Df|e6@o0;ZQtO;vRJ%Mb0v=GdW&&%^2-wLk zbO52mn?~Tg_@$nzCON5n$#xH&cs$h#+oF>F=b5Y~wWg!q+?bjBQCc2@D(k^*aRt!9 zP>&O9xydHY;p^S7VuFJQm(S~Ujyw3$;No?4wv-_LXa^@Xi-{#Y^MK{B1n>Q1gN8BD z*uDI07x~UMuf$@*-&#RB>@gV&Mule2+BlZKpZ~2{8aXY*lk0^9_aDFTs`$8PFQn8$ zqCcPwjv#j7b7YNb^>Ht>ez7~$w1Vb?F{&@C6J!4SkK4zhp9k_6OA!+>oPLq6%kGZ4XE7UcQ_i2Pz z%M|xe{4vqXS6dFMOumHS9a5wD=nXp9O&wK}wUo;`p&5$d8lmw+#7gd2TJa7uaPJUe zCuQ0y_t2g`;;UA~-k_zrt%so0+18jkUJS#p=*HBxm)Z|Y7HzNU>%Ez>DrJXb<%nZK zacjbB+nIJdi76+8Is)uB;T?Iozu7Mr`OKQ98A}8u;*+{P>WWiN-tV3o*Bu1j96LM7 zf9rlDFGy}mDlw1LHqAqG2l~U#idSncO?W7XSK(!ONjLA^4K&y`Oz7N+qWqd1VZ85t zI^_ecOl`I`t&bf2$cWDMz_YfG@3LzgYE^*(7x4Ez1A4XFoTGaxbXiTc1jkav&lNkA z9!VxGoqXZlfwoQVCb|7QJBWDjmzt(;o$$>VOWbL9^NxSBN1dJOYDdowM91!GE;}+> zMQYKBlqN~)V)d5yKjUMsev#K|zPKb};FRR-(s0&rBU*J+P~}Nr>>{7}DZV;0p`_bi z1B6=?lJ6NBpSsfBb@M84T19&h-e+an}xD!|H(`0DGC!XHad1hSO6$e<8^Zeo4W zlVaxN|0Yv26dN;{3wigI7o$$iTOhmf&f{!nxpm9^vSFOI9OD>EA40P7lVQ<%Aixef zJGO^(ko=_c)KnVst1lQdv}wiv)Ur>YZ|KoL(#>;zCXlv#KznHPW+_r*orlsc05!Ac zEqfm7>YL8iI_e+~rO#t*!)y&*Wfxl4m(Vc`N;U7cati~%*0~zvYqc(AEbeEwnTTtV z>W&6auya~u3Sq1W1Q5kvc^>#-Gjg%aKYD6CGn_#bE=vxLJ5tm)PEv}K1(%nIG|nH{ zZmQC8b^Tepe&mqfX7}E#`ZYCPw1X7pm{RMXbCI|}aQ4~;skZjgi9~06t0OJ#q?{W{ zr(+E$dZSBMw_n&ZOpJ#*)CqO&_52Ql7x~?Nf?l7qr5@Hf-Hjf64G|}R6V54FoOT_* zu20`TEb9eZk(&;p5T?j!CJ>8pouMl^^s|N*on${1!eCC(v88@9B6*W3%5SYDE{iJX@|DI53Se^I!)OAYI)TYt9 zOdB&<*?omsL}3y4E`>I_PvtK=XmK!DYgiPlC{G8`hIQF+riJkMX3^KQh3qL`_Pd?p zZHqJ0*O;tvZu1R2Rev|opq#E}`r2Tp3#=(+&Z7bTjV zM(Ah4W!~Q&h_>xSrw#i&o}_?63VYWGffUM~;fxH^4Oe!6XukoacmqkK*C>}S3MR*z zFL&3ZJ?EaH_&?|HA+)Y!AY1wAVME}@+HSF74v;SHao2NDa);<3<(3;#e_%LfJ;F#h zu0~Ma=U4m z$LQvE%4#sQ-Dy68Q<_Wr@k%UxPo@yE~+W=m#MI{S`YBDAe73 zSI|ZjQ!C9u)q~nOfedl2r#M41I}SR3NWZ3Munvf~(vkGq)nSf_RZj56&{18pbE2~> z1+fwbj%Nn%OaLLHXbkGHKY`DD@=J(Q6AUo;rWy~gUd2y>eo5Sx@>}i$myaW;;gl%jaGk-j2%aV)=?G&+$qzR&B-xt{=SPEQb!C;5`(s`(Pe9o z{Rs0gF`X3`in)ow1X37xN(@RDIwD7TH@c^sL}|>bwT|y{o4OyQerW%!0XRwe3aT zW}V?eByDq79pH_m&H{~QAUr&or8`M@_s@5SUKw`CtNrsh!A@!m*&f?<%FgBAVDsSc>I}4v?TuH~vVYz6Yep_)copIBJ2LJV_6I9MZO%q~W<4I=FgCvz+Ke5=mit zK7bzFemv*bWJu`<35Afj@aOKrf-em&dB6{fEF>U-4yKs18h^3_nZPqMYJl;9ulapa z!wjrO1ga=)xoHO~&qD#~hr2xqG?0!w^+%XRj=p1)ra8(*Jich!5{Wjj_zs}{mo~HL+P(xl0 zj^=UI?n`gtQW>8Vv13KUEPe^lAJcw4M#PdO0lx9!-tsB{y@t@Xt-!n{ddgfF zy5>{)-iWneMBh%|AG~&B%}dK0)j_@IN9)OfG%}BzAr0?tWv3h5a}Pb=g=VBS?+-U9 zP6Z2AW)#+GV^XcxDcCScAs3t0jQhGtb>oKgm=b z1)w=t#hHnx^X{M9l?C)hInU$Hr-cNDJU7P3oFB)Tv@0%JndW>2D>LR;O}DlhqDdV7aDx_!>Ldrwpr&e+W!_^= z_LzN-9&%#x1LKkaW*H{EzsLGTJa#Bd`6q>6e(>wdCZ4AAaV7ji7;Zed<^F5$^%nAP zJazYGZTuI{@grQg74?5CAoRsGh9K3?I(BLJ(9e8)CdT%tkhdF7D+V|T! zl>16>^^hJmZlXR4K`?a5ib66*3~{MU9shxW>GiyKNGe+I>;TlL9S-tPg3{6-1cusP zU_>>&j#|BC4x3^`zlR}ASGSE*o5xy{Ki@|cT*MA}9TLD=FP4MfVPHrKV?`ITqkv1AvT#etQCfDV!%)^c87PI= zG8pr3a6I&I_P5ano1P?xi=VWMcX-OP^GtYJcbojCVdxTQ;NWugw|NR_;6{Z!+aez( zCko;3WE+R)$z*J?2SYJ7V@3Lf4u3t{@-w|_k|CzL;;a*YLZnX}6_-yQX?zPq2fK#; z2(#9XMl$k@Awi+baN6*mYw?!j+%HMD!bH^B0HkJf8WoKPUVPQcOUt+;OiwB{16<%}uPadlOXZ zVFh&tI+c`9Q#RU?5-@byIw;uXiPdvfj6(VCAXEL`CbA71R`5!iQz~0*`ZP7;`5|PN zOAB(RH>XUM{WQa!1VbjJ0Dz4V2mXl5d!K(h+?bZ$tb5!)iL=Cwb7TiSyVG&`jXZX` z5LH`7d9wrHAps!@STnB8C6M5sQP{EuS)#KS?4TklvBWZUOVIaV|vzk>Tx8> z9SU76J9 zxE%ql7SKO4Ed}ie#|zuaz~+V9C4Og7d!mn0<2gNL3VbflCU)886gcENnX*>PcsD*d zcy~&ysX<6q?owdN4vRI>)241?KHU?k7W~q&H}OR?uyQ{BtZ99Fl1K@lni9zs))f`Q z*|1km%f+lZn^w%SpMg$HjdrBGpa*eEPD)tyONQZD)uH+PGt7w)tGhor5xrv<5!n*zJtEbgIhl@?gqGQ z7DexUb9^qJl0CCoU5Y0qa2)k&_3-FvEnGrPkRTl{DMDNmuv# zX2nk@F5f2QwKiJS&#HNpXC)8IwMM@emXMh6P#8!%8(F&*DMM52RS*CJd+>a(LAvQ$ zcQy;2|$)|XedPm2AYXZ9+LCIYQiov}mr0PCna zCdY16MTR?0CupKR={^NziP*&*B3ib&z6oJFX6VaLlj93#f~^0WQLf$x#|m|4FHfno z-eT|oRB2lTg-bJQS7(=C%8klbi~UwFCr%i)47S*Ug*R&QZm$4!NOO|zC9YS_qTj@0 z^8igDuFvKT*FpY#ONf#Uzz}bsoVSzL~0dmvL$4%k+qET$UQR)E}+J?m}U} z&8G}&KD#2@pxj?Z@L;J`2aTE-pUUQ#eT~Ic)T(&~q%O^3LSf^Gn*z zQ3Ch#2bmoXxhBRAP?Pw((N~rzq&Ay?#5Y=@eacFB-8S!vS;~ zciKMF19f>K@@a7Zwya4jT?*RdF0}k-*Zi9rvmwQtnRP1l)jTJE_Vk!Y?PZ?pm~b21 z%O@P7?}*ht*24OpzgIY9RjcDCZFy{h6ABRN&xK(XobLxPqxoqTZ?dO<>(Z|(G5N=z zeNSt0uk2OGs~=^$;!APNT(K?AkSL?h1KHf^Rv%{OH@tR$lX!AL+I(aXt#eO4v_4-) z-Co6x=+XA*o47Z5(yLb=-!KF3?;o1jEHD3l)^Oxbv0aqLlCu&_%A9iU84v|*(6l2~lJDAm!}`$QOcdi@2akAb4Ch%ZrmSJpK01A(>?o8(9};I? zk{{)8hL!N&!pgNdF00ZjHyba;LgW`l!YcEtm9<^>^$UQxYK|#jJ7d)^>grt25(Fm- zjc(<7MjN1Sb>8#XV&nh}N7|D=mU6lC=R2KSuooM_5fm%5r|}nZ6c+}*B4`a-#-%U@B+}~R1si)ZGRi2xdV=nogo{m1= z8=JxRbWiX!sR>vfgh{AmtlKa%Q)xuZNiJ`=t^nxT+x2HMkq?b^?%F$~B8hQK z3QZK3LpoIs+hq!@8^G;8;ZIdoWL$NhJk7rNrJBE0Q|#Zz?q^s!jX|k_n}Gc4dG9W0 zV~l@ksX%T=bd&%C|GEz`sJ#VlK%9vGp3XX z)lDAdGUHEXTddkr-PM5RVfx6QlL-5b#jq`3(0~7YXY>XE*1v5DoLUdxEI5$edlz?1 zXdz^wqJ<34ZPD7<`su0I){B4mY z>B}9hKLvy6VN~D4*dl&TdsU=*d#)Yyk7B2aQnU0Ive~W+^qf+^>R1N^%gAUp}ox3m1$yWTl=Zu<#zn?h+CbsTuika>n@@@Ol)j^c?b;hV7q<5D+=zTA(6 z<^7wQl+dj>B(>+d%nGyKsUrANHItIO)(s&#uYb2>ww_5Acwvht@=^axs*!cXXkD37 zGu*P}rO|RenggQse!PyzMp!;&%}AS1AqwodNGIj7fa`?&qFUY%%b)3OLrx`?lQ`%b z0>Og$#Od}cHrDBDz@5vWh*>Q8_hH(#%g|xv5n9Rv{PP8d+!7rYgJEuwq{N+P1p%k@ z1ULo~{6TjH=yv`iNhU8{F9=uUYMujK0;g(#H#Kr}cnF`?Mfb7YX0Q=Sq`~mDUHj2j zeFW9M;V{%yYJr{g+tAkomM;ENtkeUn3(4E?h))qiXAxoc41}DyeD@asY}sm-M&G0c z0j7|2GmxIktWkC}aA8pYOA?F#7Ko2Q-U)7-O2-egk`A7-Mkwj;{57HU@Xv_q45;B! z0TRRLbY>TAoV9#e_h<5#H1>kXdJb_$#{o5RsX^8%hf2XHb4KeQ^iGdCsB~3+I_;IH zx8u?V-J0utG9C(H)G`i;?U$+ppxK%otM$+X@4}iq%axb_kgRK0-HDX&nf~`O`uvJh z&1%AiMyAascDXUqGHHSbb@dA+_9J3Y_u8gMkEAhcCN^#EIdVsWZ?GlhtiZ^u$E}!`NC(V-ZEs_hnGZwFJG-u*aKs%w}N?yD1gZ9L-M}JycXUw z`@sVugOV9g@QNobdJ(B5Hny^!6G^n78Xh)~z0393_ePL+fbh!K_`f(eA$|ynoLi$g zms)memt3%3t11U8(ELdy53CQ4L@wuG`d3Kt%TAhY;R~8VJvNI+(i?a*h*>3W`BFrI zV?h=B!UwZULGxnhvDrI|XtA?fT{VWs_8(j8A0TAO>}9?x+>V{DO(MF|B%u{Mw7Sva zA0WxSzI1#d`UVBcndRNIk55)-bLj!;UU7el97173{=RT+&h}fs6w3YX< z*vWAEFplPeu-S&cdO!!UjV0bCaW{|)Fwn~OTHHs^_$GVHJtz{^CMVDa7YR7XrYwsl zpI@QLx`Fq!$Rv08LJ;_H{y)7FCwhmRJ7!pS;XH#&;(sh)>7I}a)gmiu5FkYo*h=1l zHD%>AU7(iDg_E3FS|FAH9wVKU5=v>Tkw?3(ubsMJWZyRmm-yX1Q z?A*T;sG2j=(tQ!`NizF7u*7}JWU~|_cX>Izh7YwoEwcVb`i=A5A*{2wZ$U)vmr*Cn zno4W7ZrlgN%?jCGxgUWoyNN&_4YTQW{P1nG#r7ZV3Ln|O2~_w7^?Qsp?<>%s4%{{O z?`rWEG-`zhCJC+y`}%CJ;sg^dEiM+9Ejvt)5?)NUaM^|jVx0J$q#~cf97H`T-XuRrvvsIC^Sno z$(W8EDM&lrx+&g`7%v&SB_^+Y8oc=X92r^q%xa`U$paY4-Qdk)0oJk9P8YNvKFaS! z9;#da!Hk6!q+4^u z^3_S$)vdviwT)>_bV@b56PhB}i6}%>c9A!kt;RRD$cx~+LSwrR=}on+;+&dwBr?`VK%Hf?l*=f8bg~CTKH)J_ZqdozTe_v9w*6J6Wv#4g;3y+z zvSOLxXXX5CQP+^meB_C5#`CdlKmO0;%X7O=ffGTRBLS5A{#U__JEn{u;~Y5U%w7&l z(5c$M;=FY*kB9$Da>Lta$Qjac`Op_c)zN8|xvDAzM%~_wwt6by^3h^|FH&XaX{+I|sv@~dKw*&D8cSa4} zVHpooPTzC7_eCiYan)qTLW2zylTqdSt-ukZ09;yw?SIMs26fS0m*tt$6bg*~e8^1M z_aO^zvHm-X|E%>_!|IM-R`XZe7n;3V1?mMm`%c18tRhAG5SNk;d=ON!zyN&ca0SRG z6;BtGc%L;fEu%da=nj;|RFAB%l{^V7~Cu7lt5@_M0~t+*yy4~+G=Q<8?_8v{ z*mSo@`@w$GFB~PS<#UaoAk`ppHmxJA(Kpl=b~KnTKTYicIte1UxQwG7lHt~$*#H+T~!C)b$;T#}(&6;56ffME3o$F-U`*O#FtvvC<7FcEb9^a1Rtc z)r{r&VHK?%XGy_P{EM zi6HW6-`F8`VXcOz=aP{9CClprLfbQps$RK25-R~cUad*;Jo#D^LjIb#`8CU5d}EIS zqSiI)WJ;Zcy?QM4fu-8d*uj*^G0~qpaf6NpQkOdM53*9cEMX4Gr^dbd`V)p&6iZ^L z6J=s3Mg)vA-}N$`SPna+0<0dgcBqkHeX)ub<*Mm^Tf5!!+Dn1uKr}F9Q2I+IgYs%J#{boFcNS3PZzTA-;0>&xaTh- z_N>DAF9*4NfVKTCTiAUW00ALL)EJyIoBax1sEhtgjkA|xFxf&mWKCFi1p2e3L6$D- zbcfIsb=-VRtAkl19hIGE8WTE0?^@GOln+T;+9g*YGj0`7P{yS zByF@{yW<+G7wQU2_bW6ZQP7)>z#Cfe{I}1=2nptOPSd`b?TDP!nnZ&nVONzsga=y^ zi=5Hr4K!aVmUKKwKhL%q=}-9%LOs7NnvZrWUM!_bWQ!s5t<%!WbG0`tz4MOTqOy&iD>fec33qLCS||m)P9d0Mw{!ZQoet zcl$6>zK%~WThVGTZ;ZB(AlSodOQx?7dtau99)eC`OItzgT^EZ}0v4q+A<7)!l z-jhKRy=^0|tHa6195Xktj=V*GC)i!%&?@WD`SRK`8C#3Bt|{;{y&c0zyL`M)gHA)$o#~5%)6~5|F5mxMZ@KZO@B< z(%m-!tFbxoh2x=hP%(B{UVNxFK?c4@>@}IClG6KO|M!0Uqy4T=4jkDmT~ub9^d%b! ztL>HX_ai=BcC_6{Z=w4rT{LD;1*x9&KvH);Y2QYSQtXFayhcoiA{5g|#|xHsCvXyz zt+l>lw!Lf3k9IyZ45REwa@R7M){^MiW?LD?yW&Y}eAPS_;Rz=6NXsLJdmbl#Kae~2 zr=X%AQ^RE4=y!OnTED)7$MfYny=Z6h}?f-POFj| zLgy2nD0es}k<)u-o~y5pj*bzsG!rBeJ!|OkUbq*mh$1~WNmsa!nMD#(LCojOGwnG8 zZZS_Cl#xl_emxSa4x^&TI}sdmXT2o;AsE+1PRLT4|(=WYO|n4bBF&xU#6` zBWfvfN3>p*H%v}-3YMCjmZ@oW9mNEG)w{i&4xR#kj|J-}=HQ?EhsH3};CNtv2eX}) znf9gdLlwPX-8ro^PTDCZ8AKCLxxZa>$jAUxnKJ=o7ZuF_F+pNQJ7sttOclo;M%wIfmC50x z&QHBx{~w_pHwHI@fX1W^lOZfn5_V}S(?oLRyP<%x?a}s{p?#E;cU(9s3E5y)9jn!O zX`+c5i0+MUY=|Da$iPQ@Hc#60#8sECpJhR+H#|%~5^+*H1u^l?TRp0Z)dX7uDr~*e z4Bi*cl8#K+2CMc?_*$b<^%^hry~9A$K5`9<6gp}X%yaH`gIhTZoBo@pP4@>~%%E}q zvNUMVZ|`88eQclorEi@<^JS8Q9?cII*6Plioi;9rmur0Tcn*QY;-PGR+}f-_UDc)~ zn5GKq`!r$-&B~=_>)w;QCq0y_b9JYWa%mmoho>#BHbxU2k6M~y_u;W)2extKS~AlNvg?VS8%xvYKAxUD`LQPzGl} z2fgPAu=Xne(4^h#-EjODQn)zKh4&oY=a?Kz?tei|s~n~{qkn_X_Hz1M@B}yyii)VZ z#iJVDsO27#8|axC-l2Te#6?J&3ey?OcJYfFD6D#@Wc-rACfKyl|HSnp1xAyrkBR4P7>NBFd ztL5Y{Owp8>jJmDl?{n4EwQXb5({PwhG!#bpW0?XJQ^z#pdQz+xgpd+Y>~c<_8bu6S zq*5G>V`QZ=>#SrTeN<*JIK;F!ssRNll0`I*HpU)QrI*qmpvY;6H~xOu!@HbrWMpQyxemnn zVyBG2>c&%ef-sIRbuoM`cGdei{5T8S$u0s*fRH6LeZ4(J?jPl1G`Ywb7xe}b*d4G-LfrBK`-nzxu8BH+jHz*Cyis^}$Sz`pP!GMf>QEjyhV;Zq%yGW&sY!|H0dOUK8MC!9!NOPitf zVt#3#o;b$-T|Sc}m>Md1y34|A=g2qUoI?1|no05o{xCU`_EHf&(o?#eSI%TCNC1TX zOQnJkl?BnbVt<9k;YmRKW9>>h1gE^Y5LLmbul6^w!*ZvN(boZ$^7aOD>Je2fFM)jhWu0de=)Lt9_ ze7d8)g@pReM{ax^?A7;!3jd36U;Yu%anLuj*4?gK)j%nmXA}7}JsbJL*{rMQA9YH9KhS z|FMA3&Fa;wCS4^SBM+S){i|z+5A{tm@|4@8&6UlXb*kN~6#DzwyK;FVCR6K{Kn2&h zUw;<{E%4Fa`1S|nnU@A@TkocgfB65{ZPcG=WN1rlCQ$w~gyBZzcKcG9eE>_{<8cLl#z*gkn$v+BhwVU&D`t(Ct zAdP6Cb8%uTn8?r~Tq!3HGXFZeUPrUcuUfONqJFQ%ciA=BF9Q4Gr9&{k`9`Ndu~G4S zvNZm;1Dta%Eb^H#Y1Afjk0rg<>u6^nvIz1vca(BWU=2!n-1!){iB0Rz^#7|`=8Rqv zEDseMyfyrHan70Rmna%_k()O^!323P9A!8OaP(33<-fw-Nh~2@q^jQYR)ajwO6gZJ zhTCOSWqT-~XpL{lv95F%AA`N&3GgZOvb`P{8*e9ao-YQrHu^rXEXcbBzi&pWwR!C} zn!P}b!dkkDjks;nRi6DciN(+6gIV*W4+#VA}Uk6O4}a=NrVhmh>U?@mLy zfFmgJqK#i{`b|anZyIb(3Vsd=b{gbK#ad&bf4(S=-j{dV<2K8)RHw64-+^n&wV@z1 zJosMTGZP;!GC0yVPNBax)urK7fnx|tLwh!))@Qkt`s`xrEbCxUtzKMbXT;q%NC%vOn%9{ItPIyd<~Hm365DEYr1SP`;6X$cSZVw8qTqF`s$j(w0 z3ZV2cVRS{}`p3@ME;+!o2dgRk>TDcnLC?Ck}U5Q#80ys`>rycI1IsWD7 zQ$2jV#DvmqnjoJvrT2Av$o1+1{&g!FJGx^*k)A#^+M%NO>fdKmZoNu=*TZrzw z6Z`i>6_*)N8qSF?_s~+pAua*G%;&ZT05hn0bvRc-GB(n0u!!|f z`t2WT@=9AM(V{)rmUi_(4Eiq1S(rTxaUzCwClV%yS~af;t-pd!XrGiZ1m6%Op4xfX z7BBQKW^2$fwd(p$8Ydz{E*0xrd#AJ)SWA zvy>y{B0jm6(2on?|7maam2LE9^cC~|4v^^y=qqid>4}HMbwx+fv|KPXp^P_E^Zn(xQltN2ZMp1kv%d^~v%V%4 zQk>4e{FDTZP|%0b1#9tWR=YixhZf64in9kA7%)$6`3qwhTCEU!;->gqtWsPMe$n|7 z3Zv1t6~}pf*2;*!9S1t_15J`H#xIF0ZjVTuFUHBPJgpmy2&dXEYupjEwOof2(nAoS z9kpUQMv?@(ETWwLI}-R?==ecSVQE+E!7C1za~BX1E;B4juVopWU=$F;c6TidXe9fw zG2P8g%v@%C`&aTRCbp?qbFuLWbr%n!bK7=f%17a*vU+kwzBpLNSLYFj>*CX2m>po4 z&BkNYCs+2yP6Qy!BmFmsv;ZuRv=t|2m8@Z zsm(vDP$C|xx$Ur@q%Mu{1+`i&buJPG2VBHPiJMdR%)J`=9Dl5alK)oSeF`qd_(DXv zm5DNN0ACqn#4-S^pXzrj3yC7-VZxD;UMm3m)dY^P_fH+8hnq>6vrMBcsH|Y(lNJUK zN#2{rXnZ&mtYwQt)gWj&(hqD60XCGsH1;WL4FL8=@5&WcQ(Wfx%hB9#$j|M`|E_?+jd4HXd`iM*S^+LroKxJ07QMz_wGJ+`KG%vjxE$m! zR#e;8jZx%GiMB;#xara=Wz=I))W6b%*2P^_i=K+DizUCIhBxH$n~rk6s-YCT@m-Gy zPkK7*+jez>h2gt&qW23CPG2IfLbwP)|CX%014hj`cj%b$Weuf`z#ck;CY2gwyWcFnAIjWRCWc-(_aAj zpD2b{-=Dq={AYT&WzH(i02?zy9>IQ-yI8SlG>fUduuoZH7Ysg_gB-%8G=&5iQ~Koj?1OuiH& zruOJn>;FkxVfPjT59foo=6!fXD=Htxfw=F&U8-4wg*c14A2p14>xM6j=xkUbb^gV)_WNBH2JwK6{s!=Na8?q0YPGdX-h4JEC3gb+`6*g6{IfH5(-t6K%)YmF!XHJdfZ%j)%s zurTy6CEgaldk&}saJgPl$k5kI#}!_fWCYq`s;#K=NL)uF&r4#KZuL`?n8G(JJ^-0z z`AY~?Kgv4=E5?=+G66yaa&E*@$%|iPc7D7LrX}}5c?FXAX846?7*ODf+QoeXaaI21 zKNFf~U^3;uEyQ9vq(&gFQb=s$?V?idH6g5K@82w9h&Ted{SW+D#Up_u1x|3Ge6e#% z_atqLJo;^%e0`O26XrKp-Z+)E=|~9FV^GaB7Hk-Ab+%O~zHz`2Kg|{QwWtQ=gaBc- zt(?h`n`Rs%C*Cpu{*fDUWs4>2{U%+RQ0q?+s$6*v;&)GX`Z0_Nj4KDhX%ws_?=k39 zbox*qM8W*B*M`=}7w%VI(a0^OA znjX-?o8^d7sX#Rj&Y&-3$xh7J2cSjEYpddDmH6{Ui80v5Lh{r!qj;VB(fjxNt>eJrY$pRB!wO40DerAca*c>kPvs|+rxR(O_$^w5% zg#<8UY)-J6t&^k~G3|b2JSbh`&y4GDORV=STbRio!B&Y{9p$UI&)&!UChpp)bwCsP zerR?b1p~q@!GJB&%AdX;NWqL~Va`4b{~q>E|AoDY&69A6>Bt*gj56u_W~8-LllvIc z9NY7ed($eFkQ}u$;N{L>zD#Y^yE*r-k3*0{z*SN>HojGDZJRA)3xAXsJXIOSAwSMp zi)Ar__C7_dka@}5fmVjAXpQ?FeE`X|W{mdw=9MS&uReukod>PUwbSGckII4HJlXhD zBX;hOH}Yaf%P!u}DkJRp*#<4X^c{VF7XpbC01a1bV=Fc2;#y{Rgy*OU|Py z_08tD=?9Ij%X3CP<8RIc5@<4c(<>x>FlGtLt_0B!l^zKGSJG#EPZqB7q(Rn0jZ!jI zxFOtBmz=ADGTERL6xlNRdV;10s?En^xnq@yLsI@isAkL>10he0`O2hv9MYY}Mo zOqfl8P3v^~s_uTx@Z5~bxS3xt#&1Z@U2XaY9HPFEPUOAK^Iq2_H(STZ6bkEn+D)vo z?@e{$ysjUo;TtWI;cM`E+8X;d$7hmF!(W2fiz_A0ei>J)vVVt6*Ftx%Lq^_rMNEl# z(ACg|pP%rH07*t2MAbHAppBHrt8aJ`20Bc=gBU{nzxHi*kS6h zm*jI-{YVMIv^ZQYljN_!$`?Co-{>`26`cN5^5&h4xggC;n^mzH$+uucfwPLlez3<4 zq?T=~;MUR%U^Meo_bWF#H;Sr+WMo+%v)D|=YA6RM^joVTg!rg zOx+*VAdVwc0u@qUAjvVdBxuWk2Uj$)08H2^6qM#(}6`4506z=aP zGx(JFzsh7K))fC^0TjgKBEEq|S1>a8HJ8pB$jXpy`6p8P5%!mAGfFw{(ylL<5B#_+ z-2OEOji_K^1D|4{xTCAem)IDM^YFXUN20UeOTM}@o8RB@-Y($FX5nN-J8DBhHTO*Sxzw|?wYyqy<& z6xOr8Ti}<&ClpGDHovJSZ<@$W< z(u^uTD`gjhxEAgRP5fb{V301to$hbwoQMQZ`6=57-2gGefjN)|C7dM7Aj-j z3PA{+A4Df?bo#t^B+C->#fl_P)H}kYn`YWzFt>Ahirg_ zuhx-g_Qv|oE6SjV*VCrsM7I7aE}!MHm52YXKqAn5RA?e&eJ+m#*M1HO-$-#_1uz*%*md0z>Me13yA0HZwmRtwGl7 z1?*)G(&n8xAszRRW6k%?7N$SsOZ!hw#1LwADS0n0uqQJ9y3J6J)8gWH2@&Vw)3zU` z@~v6!(RgF6SMm-SkuL7Zq&e;^^_J?qHOmsT^#8^L#T9KOaJ6!GDMlj&Kdde6+ySfe zWJn#NM$f=`vhq16HrK+fAFj>>K=r*++U2}$zDYY?o0#vQh|VeYYxPTrdW_}_Zl%7P zqIph3QPQaXCqmwlp$2z?M$2bcuZnf98gzP5ZlRIv zwifN>YK?@Q8lFpDj;^lUhOT=kZ-UP2Z&+^thY{t1$(x%2amiPe!zeLQstV>shpQk_ z2qW)VRJcOElM!|&fOrTRs2{=*mtUzsF^jA z#k1;8H~!F#hhV4Mm1aJw!@qUuQdN5_jp5CH3|@rJC9bsxM1}v;YaE%TKf2pnK!V+X z3UzmVVfIsp$d=8ZdbS4^%o-~N81r@Z84eXsHUT@VV z-*P-!i2?F5A&E2dK%${xo*;Obc4i5v!6ckU|;J`u86O+b)9Wn&V* zCDypTPM3xxa6fvP^MY>4*QsE2O`l0VY#AGdCT(T8)WP-~QW^RaM4kf=R_owWyITz< zlQ6hOk09TO2Xq%4c(!a5MAb3?K2t`~LB)eV!6C79^Rg{lRfr9M6*eWHY)Yd;7ia)E ztnzftHrUjhs(puSZBra>jd++_y0=WydJrZCXYnkqbtC|N$MIj{2g>yZW%KB&s_9>x zEck=`e?sh4;^+QF%7y%d`=sy%6D{W!FKwr?s5;ZNJtS;dIX~IulB&Lo-w@Dq_VDG8 z2cUBSFz;H=XW(L=ldiUusi$s_8SX%4Lly(3WSx` zkDsPnUD`FY@h*Fo1!d&b^r_+54v8OfCh4-a4N0J_FGuYP5%^?LP0TXfSmf zr1;(tW%FiCS)|mZSc=@~4CX!nE{M$@R z;Y?wJ)O4L^d;Bl<52gjxplKF%lutBI*Wa1ICw_g2`;{7WIdJLZd|aYdtN7{By_N7Z zG2NNG3K1F#R1safa#nc>d)X$xZRkJTKu=_6`ZMy!*PS^m-ED_TZ@HNDZqs8;nSP12 zDp&PHy0ZC)KELVkZnZ!(C>Ui+9*%;h}rhaSF0-G5GnSE<6;x8 z+%XV3K34U0=1_9buYTgd8S+1z_D2RoyN`F#%2CQv3cW1@gD>pZ(5MHTNAM zAZ^q9C0aEc?WAlZk}kH)oIDv;#j%vO32~BA=>@yEGEkD2`xF&D3go7PH+o9Qz(4;HL+jcaD0}OJ*(9GO`&vO z-p-Vs)vnnDb+w0dM^Do@lBQI(jcP9^@YXVKpJ3h>bA(BW??Wcv66n{Lwevw_b_)dD zKM|%&B_AW_4g?)sy0#Ib)RftAelu;|OePA{|ArE!)WIQ8L1Y+8};wxoghpj;{7sGXy^SL=X{A2*~(z8WoVT zp~W$VAb12j`0Y`{1l{Y0lIQ+!+Bc)at{Z#`oNoVPGyZ>?^))4XnN_jwo%K0{6ZgyU z%t=~)aE2CrMkFHken1l6{%ZGvsdoTB&1n^D*g}VIq3~-7xY{J_ih$nrX{*w5)6757 zkpBos7Eg+0ZVvK5xMet;I)=#J1*J5jd}$Gp z**>i3+80|YdOxR-ha+_+z8)o!5lHaw7)ze>5d->t+f<*a8&=--4AV-(_4LvOkw7nV zT}ayjE(Tm`1N7yfS_0(gW9gz35pOu6l0*g&Ef|;`W)c`ri&Gz2-R6vrpsgNKJO)X_ zqCeYsej#YXSrmlWnrz%Mw7m~z*>I!y+~PV6_N|L5>)t>Af%XeeF?io7lTz*kUtN30MGpPJ6HZ&Y;20h4CdL{37ZizVja!3)7Uo^E)L2aFuq;Ku zG}vq4=Q)64wvZrv&;q!J92`%r_ZQRFpRlbB1)=9XN%>RBQxnK5#TfVJT_1-@{HI^eoqW$+CVa;m z)a)|3D~GU<=Z!GUfIfpHVZJ$LF+C;>=|-8q1>!W%*HB7TQ=ziI;bP4LprOi2hMiT* zSFJ%XtOZCW5e1Met%9wh!K{mcqLCpX4JoA)BKSNy2iKE}SANLsu(7u}@{c*gJBTeZ zwtyxTB-?-zpuDfL_^_UU;#~N$hYj|@F_@R7nR~wh@O+JqA><$Wz*^0rqBFZLler;1 z8ZA$}J}*3i3ME~0)E@CEY|>iR#8G7@Dw8c#K*EgQ=D8>n<|{PoHPM3ebW9u@kTtE5 zTR{JLA&4^KccdvtlI#&;x{&-uv>`J|q{QcR!a0E#J9cCs95S6W-&B7Klb`M{EEzxE z67YR3bvF!)GWVnu72As*4|iWeU2E{kqR(?avECW_%GD80FkVHRBonNj7elPY+r8)C zmk*~_7E6HjYr48?_6k{VI6gfkJRyLn6mfiM>?Y$ydbBg2u`0W%rg47A36;cSZTu$o zdkq`oK!|i4tj1ZCW8svv65h!=N;~f~0i9R*|LxH_!OJaY!!E7$9zvsy^Lq=lX0)6C z>+tFic8%tU>Wd>juK_pl)`0oXrFN8g_@_k&jo*@I} zBS4E~d&fg=XLzs-=nl!IhFt*3<>nJcP9*)mS^(>BbwDnectC-JhM7mfhzhXEdCDet zj{mdG-b3BX68(XVe49bd%kfY`QS55b^()WgeIbrcv8M%b`eM9CNwkcsSp$OyO zN5?J~FgQA&IYqq3o z^&!u0TI_T|us_z*d=Hzn&w9l@{*8umFe5HK>XvbsJS^4XT>6{RbELzPsJPx?WsA%` z$TziXSna8!1I@Wac=wJei!;Ca=H`JS-d`p7TGZ(5wmzMkuYo}+H=Jj0N91j2=MLbe z>1hs77a*0EtPu<{p~(q1rxe^!JOXRe^zfmkB2cDljUV1p;-QQ`BXynqVybmaWc}x& z`C?sg_)V#c9eUyXlANlcYr5aj@}Wo0X>3sR7c)0=N_vaoDu#I{Hv}w!g9)q@>S-($*`L=J?mBf?~aE(lDxNT;{M!S(K zb?Ug407_=_>7XX6x}RI2HSsQ%+mj_bq-%{NVJQm*@yf?kFri2tGS{QEiCV(u9!=-rhGzuH>iyxjsV@)1 zatM@Dc=>CLKNSW9iz25tyW~uKBNfS>|J~!$EQ@lhQ}%mxlW*En$iSGCSsh}35hm7M z8F6EFf0|w=Fim+{Z-d^IfhY;Z-1mNcw10*<6X-s?gP2iql4CgAe;Q`)kh0>LG~*8t zvR1TjqG0@1JGOPjI*hcTnLt}zw?{;MaatyVMP>WO1bQtpK@$V{dlaP%HGgj73fH;5 zIBK8$I16T%1a4n- z-%K7LceZKnt8tL2IrMF#l<(i3cSMtGams2S41mJ`e& zeopQ{y+3x7ALe@5kHxx)tVU_jz}W1+E7fU%8i7VRpAhdc9;K>@`m>*44H)|z@E|5Q zyqjg4%SM}69&geNvwP_dZSaX3G}g@%BnK$h=T+w%$eUW+PIzVUG;PI&?W%GB83?{q z4CX(W#I?LqqtD)Z8L%}WyFc;qaAvzU|01ZdJhMM-UDb?5t6mzv&b{yVB}tp|XY5ps zS5_*ALn3=02<=O7xO+d@zXx1~o*%rt8m-z2D}2~;{E@SwGpAjrr(ZasBjT=bG&qGj z>ybC@0ZkraR^;>^@CzZBX!w04MHS=|?@daem|o+yBngFiDKX~Z&04W z;r?FMpoQ>ZLx5DMF}i|{m9M~(e*(mF5GEjICST@{IZs+&e>)^mbpAL={hc5mvvIz|;~j@UDRL}=i*kHC?%skA@>Z=n!pgXX#H%DK{)YY1xdBD4vNXO)HhBD9R^ z-EpRK;0E!6CZd=%-Yf|;|G#OMO+_U25 z{-6@%?B{qQRQRg=O>t64Z14g2E2_BEtV0X7^TPV9o;a)a%?XW0Bf$O%PR{BmFlhxD z#tzArgBtjil%Om0JlCM2m17nn<^cQ7H%nh{<>3jvIUS^#*_f!1@j^}uMw0wfZ>`0n zl00Z!@^lMTOggfb%QMIZWa4lXnHo)7ace*${4+03)35_l`8Xu4)IlC;Hrer>bi=Dg zZ`X(|8oLeYG;O=T9joy-p}G?7TnR3uhtgtob?C{XFKKoREaf$XqOL1p3*LDfIivAI z4fMVLW2UzVULxs_gV-&Ts>h*yXc0*byMDW_o2&}|EN;ILnLlRvR2it{W+Yp6|A>sJ zhUTul<=sdN5%fl@p}e|_pn2pv4X1(w$BT@mn%nR_$1sxKVRj?Ze?*fv>IbhT3*)*$8;L#X42us;2%)p3UI+^er2i8oTGL-zS=Ps@bgco%52n%@#lmqnkg zL*E)p_(4WN^}=R^<@+(Zq~)5AmT06DZj! z00`SqkE2gH)c=^K#Di?jDu+Aw4N#~azCRsVw&Pc!OdXvrb7~nzEX$yyJ{VizhX4G# zLMW5!aqp>TZ&&oaY+J+c4Jo0s@XwN~Wd70SY|2J}^RubG*#75@YKwF-%Dgbc^-10| zx%*G)rtG{O)?9#@OIpFzO+)GprWrao2WVE=sJ0zMUHHja?|tKIXlyxC1N~>WcZx(k zr1Uv++}%bBCh(_*9~{*5R1j$RNUx~VmQBXN1kN5QL<)lv$7SSqr1_Y?6rLHWXG_$( zvr`z@0{+JprtAF$wI`~6&6IPfE*i{9Ob`fjk6S-DLmN9+R4eWjj(T(q$?}Kd0g`32 z6x~OCO9AJ^!u9s7iGz%AQFUsWuTd+N!rB9LN`$iRzWBbg%E?9_V9)zb!#=MrGUhdL~)ao;5~Y%0xZ`g{-w3HW2W(ONr;CFXW|?oK;= zH<|;Oz+X&m;faazK6@)ynCl)xinKt8riZ@d{_v|q7=zb2P-KQ>jU-+Zj%Ae+1M8(C zmtv|)eZuN0xn&J5Dt*c5x1zmKY!{Oj!X5d zVy)7lIkBIkkElxAF`ha&F#wc5*Xo_2BXB2oKWcZr3f7x}s4zsZB1@>-UPb#+JmpVH zdh>cmN8j9lrX;-v#0aG9*o^3oO1oZRMX3X;iRmsMU0wZd}ab)ljUa5%u^TzMttffvmkGe$>cTTmD@1_CF?EaWsln1#Ntf*Z<|ZeqzbGDj~F-O&@VI zRx2T6`L)xV0zjOTbynPD1|>Eq(=V-qM7E{~iYQCzelKZ}4gXYufA0QVq@=-r$gcUa z2++H}XhvG$WN&!VCOH=53>+!tw(tPMk-C->$_3)-XDU{yi#%!HxkjrTq!$w>Fx!|2Qg&+FW!rH`Uie}+mYoIUXbD~wiYgd;+5|y=C{9AxfL9cCH&j6#1;B zTgNc03KUCkf{kc=#XDR(>bd0H6~qw#(&$?+c3oKyq2o5cY9ZDW^0_zVzc5T5E2DVK zJqB)q`UF#dx;doxugg_8EUO}Vv-eH_oW#nk^4tMewhU>vsfqH|CVv*!((QPSX#;A9 z3Q8O_`6xqt_KydcbYoW-VvdbF;?&XyDcmB)& zRYJF)vP06oa&$m%eYm#FE$bvN4849_CvxVI{a^9TCOJ2Cyi zJlo%lp(v8hqkczobKSzyndZM@kfAi4^{uIacu0|}*Q1s`HzfMyn{(}LV@O#lT3W*X zqBnuOjrez9T5Y=r}njly8myh z^nB!ZL~70cmr@bTp~8(UGF-9~<@?ur@_W%`In`zdXQlACpG4t1(QtLRyXAWUb%lG- zrvq0avC1SPv=VdH#*Q@>6nPpr*VS#>ZP<~Af$_XbUizb!mlVeShzrOBJrm*^+qO)u zPFJT5-mUR*;Z?O@^m+n6tY)BQiJp<9GRf+R(PIJ{yHS4d-+|443<)6)N%}kRaBnvb zadAS>2^}*rt$l?&NH9W(ryG=P*Aawtqu-!vI9#%g3KN~GJ3T+yrPckXBc7)x)=xIP zlQ;O**NhO&CGl*A;pxSH^I&|=#?73oCe*Ow!VsafH}1I(8#5&n($;tr8ybH#-#n|` zWlHJqh5JmirZLe8xViHAfMMfC7|(5;W@pDzJ4A=6 zT;XG(#fO(FLmYLb0iCZ6I|0n)J&@WdPixq#wmt)??A+SCx0mx+gj3cNGbjyF{$El` z3581UhRaEJjx3km)H{^)Kz|;$89EP{JpM7epdV>Qc_9-Oka|yWcHE6WiYDXziMQOa z0>)D2?3lba6Ae?+{z!#_24dS?8VKhThSrt}@Iwh-8{dw;NwsMb&$)ZU1gH)MZK&6S z-UC9_mBTA_qqtH`+PvQ&-;QCwO!mr+Eonw^Z_p~K*k-bEHuQS-j#ClS2guBlY+mDs zg`J|L4~H7G|D6av$zCj~kwl)vJOGVKQshtO#p;Z$&+8tBm+{Lja^4(R{8wmF2JE?? zgo)Z^k=hv^`&T~K7xH-niz@h`_Bp}+M?Rqh?|DHrO*vB?!tAfuRPBqei%ZweJL_ck zfQZ<6Q;!sS8)yFkem&n|pZf2A`U{tbzuH;}5k(dh*F{v_;c%a+42PCYv_ z7l3Aa{x|0|owOHUOG~PZ7J5s(ZKNYWD*B*s8SmV!heel4FYs%bu;pWK%tb0?M$`;= z6k2b~X`Qzsyg2F zYSsb=PpMG+_D$-RXIJ0vrD8jX^ZGQ=rq$t;KS^X~fYZ&3OU#uneSWb`+~Wpt#Sc*w ztE%$z-}T=c{%rA#D(T>eV~3Z7#{%U)NTG&nO+7H0vM@Q#(*I7GZ3{7xjzwbQpkfB^ zWjyZ*|7fgV_9NM)!C-vp$tC&t@=ic@+syUT7s*Z)zg1*IyLYruLV6K?fxprvZY)`g z{?YV`wpuButU6E)RR^0@hX~8QFFpJ2QW-oA%omvI^<17_sC<3f`J>ZtIY|PYdC*9q zn;LEIOY<0%HeNZj0&)EGQh7|25{)R$T^%}J9t=wTGH&mce@AjEgD;rVHbTQ0iuTb< z6$d&^T2ORYC6U;-*ipaG`BH8M$I_htme=f27tqxz2o6SAKxLAKq7qQ%Yg9l9m8eAm z#Ch7-`aC+W)0BR* z&1zzpxqPWjDD(C)i-mEaUAtCj-%{(T+>>OE zvKnEH?05gL|BX1c_ITXC1CfS{SV=Akj1j3J@uLQ_d+BuQ;Ew7P=U#fIK5huq2ac-i zw;n9!hgr&*J6XkJZwm@T;=DipCGb~n%t1p)l&?0vy(wd1y<_plSOS}#`X4=^l2ij) zn;|B!;e0n18F;6qY&|}ar$3+|#EK|c`)ztB|7iCfYUeN4st)7-V#j9wk;;=Gc~{%^ znHsNbHU`0Mc07gjeVw)T{?7F`r%>m;wNT@lhw|$C{oM+he2Hzn9uw;^BAjHB^4~*0 zOs)_?D@ZMV97U@kr;xUS_cRnH4rZC+%}TuDyoF=u!t5kG3Ls~deDUpGBN=e2&fJtj zN%0S=>7E$mccJ}RZ#;~9{hb(zTdsCjOyQ)5JDZWuI`^>s7Q1@<0@+}Jd7XGFnH#F- zVoW)EKN9mEbRJvsC+`CQ;kGKJ-BT~bJFE8PEFsPN?XlX-Ot|B`--(5g7wQtI zRf6Xb^(6hewPfCtY4|=88Ym)Fr6&z4xcTJkBVqpw!={f-R|VEQJhe(D+H^>9f4S`T zTKr(Y9oD+yz-@)M>`XXx5i94hCt6aJv|KfiBj#3R#MceG{tACKz4*oLx|LOxbc{Lp z`DXL!W~EU7k{ymwx-R;L<@U)#NzXj}f7&VJ)-Cf;peG@l7JV6yW6=&j)1-qqeGlW=e?G8o5Suny332j2IC8Xp1{D@44ZojU5F}5 zVv)Kb*5QnoqNA%jjU=fFldTIZlEbSwsHAKUiVEqs_0L?Z{!qU*B?EM-|5m{0w>e?n znww_G-SFpdM|{Q7F8f^ozVq#o|0K*yY!@hpoikExpNYDyV0M?IV|L-T9|V4dy&A_rfT$su7BQp)S5}4r(>7S3(ySVL;6`|3j z=|(t|5Z&dQ7Go`h(XU-vU|G17saBNdqS*!0NHfdnN}!Iy*L?5S(RFw7plmGJ1`7=z z>IAu*(ZYNV_)1(zo~Zfkur?%E@^X+UpF>u~S($Bn!KC$hWBf;r(}ZuSfD1azZn{xy zC0T$~G9U*{k_7?9;HRt`OcB>qp4kx>IYHZGKujAJR*Fp|y_&aiFQ?f5wD2o}#H5V9 ziTDR#Ycdw6m36Bz#HTPVywJN`JQOn=|<5~@#yeXuu!H=*$P}bqxWfI@q z!X|*=D}{KkGnbrZ{Fhh=U_d zmJV2AkO8aVEm_FtzI4ENu)HO;5@gEwVEU+*IK84mMno}0-2O-xS=Hdg3D4dcqCcBkw7h_(s&6ouaOM`IZaqy`TCe9kOeyI!^N>er%at-$GEhcNCC@ z)2A#TsFS%m!_(YR8UrI);<5kxZ)l57Z=_y%D?76{rL50ll!5%#t)~T*7nY(5u(0`w zaHA_0o#QUOp!IrX>{f#fWH)vEQSe~$IlF>xf;UTXMud2o(s!$^rivdK;&|sSzm;*$ zU*(uNKlR#qJng7q1pHtRLB`7N()u zGRnpYG{Z?6XBTT;PhSBJ!LuGBcl|J3(&QE}tG>&u#p zhGm0UpB)%W@0QpZ6lo$I+|1?MXR_PQN0gj&SzYbDW+0OOB?os2fnqhdKX<|+Wrz!$ z?)XtXZehsQ-EIpBs+y~q*Te+Bex3Q?_g#M;PA0!ZaIGZ-8&OaO7{$F!2oR(Uykd05 zYSJ)-8=c= zGni}2HwbOKa?b`;2FfRnU(V$(4GF~eV_$@Ll{HF85!w%18XR1g8%*Wnycn#b*kj8n zCE{k-xc71h@Jkp_CR4*+DJIWS(9_>3PS?X`dz$6fq6Q7pgRxD>HbBP*KXg<9Ju-(zQq2pJ9u48S|Lp$!vq-eZ47 zY1Ytv?oOjZ-T|sNCs9|Xv8MrzLyheZhoM9CHb`FN47XLF$k^dM6!MYkmAb1A}t`1B3E_gZTHQp!n1P3=9oSRzgg}r{Lzx|K3aA4L8C& zcu9eo<3hoTI#hvrhLTMIVg?JJ@#_u92Cm8$XHr|4m9-*H1xK3|W>7a!LM>A_fPsLa zS}!3l&xGDNOCGBlLwiLcXC|vC1D6|OX=N6IkH_x5IB@%MGuZd>ExVIt>BV=+_rSf# zv*_EwpF8WHMuQvG+TP7`LBr1eoBqQEM$_cK5)gg{3_c0n)^`2<%x}J*a(;QVVrd?D ze5^M1e^<}Zk;o#08h~?OqM^@r&OXBSNqboHGe0ff5C>X~)AWH1B&h$)DEzVg+9ZY_ z_~DC>2OPR~k7znhW$|Nfi*|}s_V{&MX0W+*@dFI`6%n~{zYC#K{g*#7DtVB+1#I56 zGjiSsh#FJeb97nT<{7H0$1;+9NkK9aVuBZ?@Z zQ&h?BtS}le=ab%d%B5j(VW~GtlEByAKI%v4_ll_=wd{{0JX|@V>4bQD#&E_~-oR!C zZ;xJd3K@IsbR^2tCSD+XYR&9H*|+`pUi%nD0D`nl4a1_PxWtfFP))srdzqDRCk;&r z3nqhJ9E#a0tw@xUb?BZTt#PqVY7*mxt+4#wqd3%4IgD|`^)l@uMc;y5QsMz-&`C%z zuj8AJc3Q9**1X%{6>!6E^G=GyZH#4sk)F55p$h9n{t$JlnIZu9#O0R|{}dnr!1;{C zM@_HH{*1TNdXp#Phv;YF05jClQJ5c*`=dhJSuC7N15x(7YG;!q$}kEcz3JoR9zY?P znbgd=X*$reUwwkm=FJRy=s4fsw>`rWC<$io+T0fPUc}lA-b zBWn2kE~!Iy4B1pdr(U?3AWcYgL2Ps@5h?q3I{&vPBAw!VAE#SNR8qR90;xz%&%sdW zYmpc<>vcfM?eU)0ZOKf&6x64bbOEi}g=&(aLHfEh^{45;>a8l;FL>inZT#4c5Ow=b z<{Cfv5eD-PzKte)W8M*yB({dh)a`61rnDc1-h9$uFete*M1uMS;`!OVQx;9#XTnaJxE&(O5#_?#$@!jE{L{rU^Rw zE0djV5?_3?vy_oUXok(Z^16L0{4+-vp!3>zr4lQB?YYpv3BJN;?~l(q7qQ`BgD&sa zq5Zz{SnSEvIVFW8#r&7NaP0s(cIk^chP|&+t~jk%dXaY;%}+ZTPgb@pn1tY#w%*i)W?Q z%}qk2u1UoifUdPkPb@uga<$#$-3dD%1a1-&IO6QnlK-&oq@5Xf0Kw@zw0<45)OxLB zo+|Rq?%%HDu`1U`P`lMz?&XRud=y=(5?o&v?sa+N?7kmW{QNf0b>QZL?BdT(G+o+Ah zy?>I0#;Jn~SsJJoNeV`-91dbMcol$?t-Igja2-xet04NQC_!ij;`wf+8!O8m+9tV5 zc80y%6!=5%Z%1~8YL)8MmP7<~K#3Mm9Lqq?LVyv>X8Gg)G~6Yf6Kds%dTX@Zsw$SyaEbN(|V;P3N!m47o$DQ z;yEUDvLPU->5(&sv*T9m>S57cNof(NH&Dd$3MfiL(Upvc(zKvS)A*$j1Duvik5R%t zRL=-V?t=8uTfz1ATFh0YQjXZ^1bWaCN-&02YmQ5*oG0@M+2u*pa;J4CtSfksdK`QUUc{P~2B zT+)sj+;0$%(5KV*TB)c58A+~vP0`}ah3C?t(b#>UY6-A!P%Tm^sQ3`~Z1tQ)yC{%U zDaF2^k`U+BX2%mZ#8OdwbjUZPM^mG!Zj~c#C``?_$m|=cniq`9(jF(UEc1XCsSC%% zuu*Lnd{OEHD-tM<=}VrP22Zs4FuKz?H5i6h1aMtUzAuRFF65ZY$8FKKyD?Er@M*T7+|a7%#{ ztEQhs?qzem&tkS+v%?hM85qAY+4Fbki`*$al(ml99b13A@B7@x{tzYQd`j`kxSO!A zkeL?P&XFZc$Ds4iO{D&Xer_nfpT@wy2BDiwsStp!lx@eAOuRE3GJlnKQ-Bvgzw!2D zqCh zfxkchQ?Eo?zPhoIporyteabg~&xKvh?57=mrF3cs4S`ZJVlqQ)(agvx{+kCc$Bp3d zoo1iImJn(AAj4B-CfMKH>_XOL9M8GOC-zn@#lgT(n9W}=E`pOz`GyBkt^G;u z+le~g4aM@x?FMO#iuXk7UFGZj>Butr>))H*?V*}%&RL`$vJ3t8ik9e(6HfEku7|iZAM0(*nDvd7k!rN|N#QG? zph-yfR0!21VWFp#;Pue{r~x*TgPOgmXHlHk8AfC^8HAab&J}Kat!nf<5!3E<_6XJ2 zFp7A|w}A!=u(og+O7%&fJ2C)6FDQQz@{Pv#S+DDkvw@dk690yC3}oQ_62nlEPAlY< z>D3NrKdSQQ?WHLCxs6YIzNO{Wiq|G`B;>kh?7E(6Y0AjJNrE7G*(I+3(H zLF1D+gfSkEX=LHELO0MeI+t3;XJq;-)s*u}QARSsWdgo{&n~4bgUUljR)W4Poo!nx zTHgb2h0Jbwl#O_W48~yfJq%JFW67(r6O#Wr58@DF4|AzUQ^Exby~lJi5`!PWJjW6j zPbf1*egaYb1i?JY=FtP9X5KImH?Tp(P3lPihb_l;O+~?lF&Joas6%E>bHz7@)yPVc zk{J6j@Nb>{eU0RCHTRm0?v`lmIjadi z9e6Bt;8bGj$j;Sfi_MJy?#>lZk2AL_4aua1)?is*2Y8muqF{@ zogR7FU5Q<1q#EiI++oA*StnpkG~C5bd8NUot%a&LO6bH*S@78@}h}^ zaXtN!F7`||s*ROGzmc(tcv~?(6M2RVE7s(9UYz%*zc%kXDh?QUa`~LD-l&-Zt{w*m zbZ5_yGqsr>YGrAWvMC0#e3*fZH0e$}A2mMU{zwE@ZQtJ`CC$o4is@SEZQ?~?s40RX z?D*=Je=k+A>7#ZeCkbMBP_!~K%_CeI064iK!{a_~8>x8T^`mgEFml+8k{>llB%fNu zgdj#sQh-|&J|3k19Q`sAp^v70&NT&V;jB;Lk0LXU5N$SCyg=w$t(UK>+9Z}jMQ}B& zs(QtxT^N9)(GG`ya+DWUaWDd)*|82N5Z30LU|cW;?TMCu@v_i|gS*{Fh4>t`zN7zR z5M_1p_;JLlSmu3DHQ?&t^eV>pKl|P+jSnmk!r4kOLPYsU7Z0^g;A(9;e^iKj%(?h@ zXW6+6dnE2JuRdOUPUGpef-B&_PtNCK!+&w`JgNxxQx1Or%|HxFR)U^z@sWEZgMK;V zVBDBzf_hDH-f0S1=$azPm3UL=P_(zl%>)i9wzRX~i3M@AX$yBVR*N~0fP@e~$mSZD z41??nOLC;!r%idTZtQwUiQV9PTx(_-lDQ z@96({c}DtOa}t{^vt#Vl8(^L!-LIph@&txCH9Rm($tC1S`*`r3y?^S~xK_z7(|4Km zuXkUrhhUN8d0a0K4;lQ8n*HYIzTQy$KzPv&{4_-&scKYa+IgWlTKJ~g+EiBM`GOG_ zf_yNS-=X87TSma>a-<6%H&E0fGoZF(+XBEs=%@QJ59b;{LiN~3F*qft7g)IuRbOz& zv*}advwGD0N4pk_SebGZA5#Q-()=s1V4OT2s0p1GFcbyS29saK@&T8eqz!ip-Avb# z_@M#OsI9;p{T?&R#~#}55SkXv!e@bvDPLj+4|oT6WsWfpb(6qU-mtc?hQUTlu;GSF zBFjR9x+XsdKp_h_wO}Gt-=sQ6IVBrLYmcBPW&iT&J(C%;U#IsryCI&IN2`s=E`$IM ztm!EXT!dlM-KB(KOy!k+OJpex%ixx;<;FrQHT1B-1*|?t;sERhWrfTwX}s6G!TCMF z07c~-i~~Iu0zz1>sW{oTt_E?+QuQw8&Mv7!22q;UFE>3s>GVA=_!b^uxXe8XLHPpM zBtg*GHthJB`Y;u;vEF?HlYn&t1vdb*QkWl6kw$NZcd>;USN8E9t`Ic`)S8?g%76%CP?xv z-EmR>oLdma!aJ~BT1t-Jo_0v)P_TFt!*+(u6YE-!X&!nNY0|ns!}kdziy>>2{}WP; zJ!nlb3zEk8RHG7kT+Af8V`NE4$|WfxDFM!ovX+M;Q;`^!VB#fCE2?LuM@zE%7#QR>{&z1~pF@re_s3*9DsN=;VanCvJvUy~{6NR5;EnTy;EO zWN?5(*l$3%p6RjD5Gbi`Mc-Bh1zMsP%StsWQGCD=b)&8QShCg6-SvkMn2zUU z5rOyLjNSBD1^@UydKvKdInGBPthRAV(T=}lT;uFdJj04=dp|V4cFzUj5k%wGq@~(7 zQ$W&jo973cLDOYXqdt!dxQEk;_W!&cKCVlZ>8Nt_3k8)iRWQy6dGTTtLnkJD{yK39 znRasSe5FCq0h()FaFSp^Uz0R#_tSEDUZCjw^JDb}1|#?;+P8$8jL7-5*P!O@DSu#MEZ|TN)S10>2aoZh))3G}`b7ArK>2fhh16QP}Nk4mk zPu?(YP}b7F9QA6Cqpv-nV^C2*!D!J7k^8uusAc~86|=*RP~XFZ>Yh!gK)&TvnrVGu z_QyYOr$wMe{XLO!_)Qft`s15=_LACX*Tc~xJkO>1yMFBmLtxz6`wy(;gRrRH!T+Gl zHu=K|I+mfLpgHvC<2PkL*2ax&(rUlrL4ppH?>wl>*-PF zQy;ph7ls$2bW*%h*lSWUlYXAWS{_oFCb(r!Y zNs@BC$|7U^r6@dQHs1pMc>T#75$XC81@OSQ%Vy+#aRUMVGKJJ)a`_nBa7IrMeV)NG z<)t>zMF+P~LPgbl3Jcd&FbaV0h20!esLH#?H!c4ylV1!CWrnczy_ST$$5)A`I z-((Q4BmwoK#nF8U9M{Q(j1id`%e6EP@$7q~85uTi;fs_3T%J6Dxh*z+f6)V**{bCv z6pP%Wh00*}Ti>i%9E}0xV&s6=%C)fH2HdlM!t-BSR&ws zNg`IcT4nt}JZbdHR`*$ntBTVdc23y@Kx0)@-)*tfL;8d-XdBH|0p^DhaJDe*cGO7$ z_0`T0_kA-()+qZS2=kFR!mKKyl{B&deIXn6Jz&3l-%RzC8loLL5!8Tl+?xfvX-dXf zPH%iQKw+t9zR2z)MxaeNo}h5z&kLkBD0eBu#7}$#N|~m?4sXk1TCXc@Bv}ua^^SsY zC{;nU8Y=T%361S~RrDl2b@=gAP*%bvoOs#WOL)U}9G<`V+4%{}G<-OWV8Rz3@hxEX zPci2k$m%I&Ik*Z`rT&tpYdWYgu(OC20qB%Xhm=8tn#Jj_shMxD>x(#b>0r;;$3f#Y z{!zyaQEU9^I7#bx(P`)8KN*jX07KpS?U9W31q&`uVgGo^Cz4ANfvN6G3wRfN`{dH_C&q zeF3Zh7{8&dvwfB&|I}YkI=*L_+QR0}H4Ty87H|c zM1|NU=}#{Z@lA|j zTujM;%v?k@C`4-5RDRaBf51*f@?suzwS2_TH%(H*5B1>Zk08 z#BAoASM2AMpgwfhBi37HO&PFe39&aBqW|nNFE+aHy~z)#N6tff znKn2$8Ken3sO$jg(?1#c1uUsYI~1IRV8Y|L^$-OD&uULe3kJh|DLiAkAN-zr7F}Sk zk?;A`jwxET2B)x`L-c2N=;yLgI}?{7A@KGPZKD(JY(-ukZ{mnv4U9P#(nqgl(cb)|NbiYb>8N>Qy$SjxA+W32Yl;}eh?QiH2$2R6o;ed6r_|;{ z)jq`~p96MF>d3;2_^07$>NvS03jezJja{WHd{{@055v$nvgiYoTIdNaUnoPiWo>KENcP|1K#6f z*SNF~+8!BVF8FNtijequ*CW%4_mQaZhnR#5k_8N;NECFs7>U1L|CJO=uIQe~Tmt0e z3%R)a=A0`9VI*3A=SQ`7X4Es2X@KU`V7`CE=@y`WdC>b2&luT&ehy^ui3dZ5)J{n7 z!poBy`#vAFbfozZ`1Zk*gP1xB6V}l|0BL&h1!aQW1m3H*V~Rk^^&=Zq36mg$dn>jr8t;*B zMq3~Ozi>UQTVlV#^%a)Rh}`-oRkwEj3^Jgc>pri%N^f2SX^x1P0y$3yF4}Ks?WFL(71zrK}a}_2MO<%GG zZvV8-8SMT*bhA9C7rf>}!j%Hrv0jp;qe1v5(X{BfdO|tI$+k~7;Nw@H3DJ$2S+6*j zlmy&fUDUCX+cT5CQg(UZ)s|svKN>>%)3~l#KMK72Mlw&B-NxcvH1>K8ey$wvDwIe4i8=A+s%Hv zJ<}7TxhTGXc!FU_E)@Jp|l(?l&%ArcNgF&w6MOR&+XWYg9{Mn z9JgGRI7B^ZbN9cCPTZmC(a}%J(>Ne>xzoh%z* zlqx(vEjCQ5G3hG{e{rnl`Mw(|*&SbYZ8tHzTmQbF`3Lz-E>`kogelJpK6P~vH!!FJ zSPzaOe$pZ4)-L&KLEAgl7X0L*FSj*2-pn?qczN&JCw=4ipwqR~lsI#0(7w@w2HeZT z1|(%kqkc(yK7{(IEBK~f^K%_H#ahzbGJW8L4Myir3~gq6x!)`5=?fW3?p_}S?#oN~ z*4>@{(ET0^q?)hT%DnVeKSowK0i;@*B>bHEAeVozzmEQLuQ>c7g`zCwKh4g7EG!nM zo{>~E*6huOlw4(bc4r}!{4MO?eNfiWY9YiU232i5v!sp`R8$yqMvEj5e8my>p$NT6 zAiOrg6g)a+$iY(TpXQ?+hp{~bZoc{^#VlcMG4I-AK+vqMr!Sl3V!YulOS1rCV8)EE-?M6Cpc2vQww@84_ zr#Ba+8A7Gf359Zr=;bGAfI*_7#XvQYedUjeeh|S(!|6Dsxl)0$5JGXm;fg-G^2L<2 zv!yzcgw)LEbyH5*QjQ{!7K6JQhpW$l#3CGqD?K0@4a1Uq)ESBeT|xN}f+bxsf~rz! zCX-5WW|&M7uNV!_Z;uP97KOt0xFIeoO_1E+G`~=*`L8~cU%1BIu zk8>fS)ar&K!t2D80d1AUBVi4oTuUS3y+p?%$(dJ=Ak4jX7pGlEQ_TSB1wLbv5I?Q4 z6?NNC7Jbc*Z6CeSgqP$jwn?Ct{-Y8Q1T(eDEnuF(&&HXsiX@ zUo|EmniS15Fb|t8)gK*ot)8K9fxlsm{L?CJ0*k zATq6z@_Sdzvx!&5vzD@*(y-Rddaf)`Im49Dhfe4CiS-C2qva-5dTe%X?ofEEXVfSQ zJa&Ioo&o1H0aSzPKhq?E=T&L=ryMja7E^PzO-K0idh-{gB9EBmTo=h45YOYa>lSyRGp%six6@3dCA9CzbwygFxlK+2mZQo=oC#TKaN_5qOC3gKYk4 z+8#rcx@0!WR}@k!kxa^1tXd@PWQ(x*nFj|FxS^$A^#WQCl=6tL?DgV{T9=9Zl>NdLRFUmgc=QRD+p$RD-gJ*CHo&XaQS zcHXdmQA!^x6;Zr8f&YgB_j@io^_E})7I1hk-ik!(`wP9vW{W!K?-iE@Yd z#EWo#(J1=%^Oda!|56Emh+CoicSe2Z8iUsSvFyBTeV@VE6Y}Z!>&mmbpw_nEWW%s` z3Ah^^WyxC&m#=Kh#i=U5kW7QlGSz}mcua+-ZY z)9h$16OI+Po_3r(9gvv3P5*xWHo20E9BqjLtSSn~O|(lay+p!-XF1{GB!=;T{E;!@ zx2uNkXyr;qGZJiQU;WW3VhB$m=D`h#vokCzWJ>r{JtExuow!teX#Rh$ z;E>!)=Iueyv8|CC9B*?@(2569Fmth!=7Nh48VKc*NiqE~A36~+uc zi|V3V9bu_uM*``%fM2a6`UX{F`Ux`s>g!zJ;aImrwjf9lkpnB1YdVP-EaTy`iei^IO#YA#W(}0el+2abq%wjs(-X%t`cg529q%xkxep_ln`uBew z-CRPkE;ROekbW0XN|uW7ba9h$(?%&n-3^V<1c#lgnUuu(GPOEmhIW`#CKcr`X(aA5 zhQ>g|8B@v+F(!z|@xBP_z#HF^jsM7>Gz{Jsq&G)K(89zSrZck2f_Wm!+lV|U)HDh) zBcU4UCIu(TKoue(Ih-smA2Bt*DjQGZ_?}n`!99gK=tqo}+zy+)c;!SJ(eQ6G-4=iP z)&x05hJl+YUhd$rcArg@T_WW5zof%9UvNX|u}&2Bt9C~EutLi(NJ(E^(U=GtuNjFN z8Kp+4`+9o)=GE1t~Ko$?T477UaW+{sY|sf0E~dh4$W6BV#2x!NJ~&>lZU zZ!ymm)7HDF6($n%SJ%L!=~SbnzRu>M2=YkUX(a{+F6(h?%4glzsG$vSUphrLXut7t zHcz?+eJhvpJ1Eu8s94Gx&*`|vC#!rrjxFMA7}v1*JNX8_UFxp*LbPlex2Wi~w+|(}lm?zAK^C zmpDjTjEB7*i0UlP>qVDG$g{!e)p#6ZEGK;7=6D{ggL50$+G@V zz0fn}5~&+(JzRJjAAN>f^$`9%-G`VV0>p_U`#3iZ;IsQCEmMq-uw{O#z1k@+&DxQv zUf)@6(VhL%vONI>>vbS zKm70e>Ti1)rW*DP1tj5GDb~DaN4ErTeQ$v<*t~V&)!kA;sm|r_sdx9VMxsk2EHt_q z8|mr}G<7m3Y(x2Nm*qXQ&6>;?-dX-s1$owMa>@bUdQ=a|@B{CJhA*vrDb2gKb?1g6 zDdkHfDBjmGy3ngbd}QyzEG4rz+D2$B=pI*>uU8a0rc=FKDrNj-xwp?OzX z^XVxT)5}9(&nswMY4s<8K}AXUMf?AsJ7M`AJ?%bpnl$43%lS^DKs?}=dfBK*&vZ#f zub37Xe%NC9c&%C#5k(<#w%ocp@(i9mCrQcJiCoqc<8KFRP{i5{EwJWDtoHt}gzN+0 z;TUizgy(WT)5i6e{9aLX_7O7i$odAsXQLEf^|%u!XY%sKs>0c!vw=M|?FnyAJYqqW z<+Out6VSVG2wm&K9~zO7LNhGRrjGEYU!f`-%7rnZ5(ZZ)d^ z1*CK`k3JF;kVP^coP|1-neMg_vpbNN;Ai2%JbW)X^yu)I+TGyrEMk&A4z~AKMcUOX z_Y>G%HQFW2kVWFxbU8FeYrStxB_1CfM{blSECjELfBk zHo2WOPrrX# z9qq>tCj&$_<_B&U-?8-TcEvZT&FVY_tFX0s0dmwqi5}vxjIln6MOyxfm_A)r@b|_@ zi4V4Z^3!N4PUo7C{0+jrM5Q6O+~4&^rY@re&S($y^kQCD0G?Ra=f~9(REuHS^1nOZ zRzYdxpJ6c$Yja!s{=N=J!0e)G@?W4w8}OwP#$ezXeD>bqvp!7ai!V4FBumOQ!|?tD z>upQ!QKFn6%|G=@#h&z5_gXu`8QeZBgtdy}{-UOsO0l%9OCcIRtLd_zIteIsUspk9so3C5=G+*S0sXlz>Dq{j2)B=Rw3A#6ai7=r#RMdnk>irf(#_;+IAwbnx zI`R{I2u8-K&&p^Hf2K35O6sT9IhJX4^4&cdO*a4{SLuQXIZ(I2cXYRd@yC=}Yh7B)a(IrI#f13Ro z`%thqjSZEZt3qIvjU(A_&;0q+D+Vc@$NmKF=TA-?S@fNQ`!)yNoYwO+{AcmxPnq%{ zlHt)OBPvVFKltf$XG{F7pnQ2BBAn^be%7Qbk2YtB;U-X|!g^NT2dU9L;K&~iKt`sc z*E_jd7nxWW-A_pK!n3niYx1BBh^pT_F5B@*hh!XObeuJ`ne)LiN%R}FQF;6>5weQ; z#(a!C+ZO2--S917mlU-0EkTZ_#Z7_rWGcHC0r;Fu zvDvhZJa~Ud_9M4mugL|?uni=OtPk5H4;g}85|-aF4&AW>AA0Q+EA+~Y%mWnNw#00M zq_nQP6YX{-8Ab{_U^{gkuXAL}Zn-of3E6irVLzR&cK}Z%s`&2e>vBn!bA(>sXK~FO zE8SL<8n``rMOiyd6q1DdnyXy%BUgABe&WTr!$nG1z*%b9=WcQWr=(RT zhtjp4qtCx?Kb1RLOKw_Z+74OoX&%mmw>mA&YAlJhCG-@lJ@K#NWD0>Cht=Pn;6wQFeD&8`qzh;wpZ#IY3(AN8x#I1j8p$X zm(_A#5J+R(e{hBgg59UJG_J@1n}=|~vtlQRlB1-#G=i{UOyKN8YMe9A+T@aAlT2QT zK|1!8lVnFVD!VuQob@u^k6=xPYE`3rptKyK_=iDzxCu)P2IS7hv7O4vB6BH7hJq;XH3V%Q2$j_mbV9u)j=bCfO) zR3d(Uj@SK#jQt#*+7y}Wf$Y4T@yF&k#0n*X&FTx| z&EC(ktZ@X5Z?A;wPRY+lUqRg&aXfRf&Dzd28Jk$;7dmaWX`s8=|6>6gmKIBiJ&^AX z2@`i!;aiG**1{2sp6lR&=YmnH&a<@?d(B<-0Ddpx@c$E5^h!k| zXUc6M=_cr*1n;E^55bpQzMFVw4gZi0#xZRpHB4alq!-|-m;%+I=A+@v8oXMQPxplH zO?@s`0n-t>vAmkmqACcqM2HybHne#Ovhg#JrFel+Q9ko7|FPl;@af^LzLmNL(lAB; zs9-@5rg`iwz{T z0M3Y3P6Z2_CMS&&-`5P?`KSEhk+%$s=cxut)o>fSywrWpz6To)t#pfQd_{ZqUz^6t zU-MiqJ$F4rXPhymBYv+OZ_@!M5p`s%EwY3@7=IAeG0=ye)l zcNI15Ve~6;`+XWY=``J1^1b=S<2ZTcJT!*gM}d#-3y(H6Hz9Wp6cdEl)Uj!LPiFFT zrqFC(?)r6RwcySsodfJ2k_ca5=fU8M>i+Gz2Nrc#n$UZ@~= zX|xdfCPzF7x18Vzc5;JtQuY{UacN{_NYCL?;tkd#}o=C{N6tdKNqhi z$1LRIRy=&40~;2sUTroebJ1JR<3b-BCZhdzc>rpM1`%;u(UEXWxgcbcYbtYn>MYQ+ z5$JJaw-c0Dye-|I0{>i6E=1tgT};&8pLVx0MAs*zI6Uyj7 z!r#X^#Hkaq#mmjhJ%+;i30fWPz?U<(yN$k{?PVP!u3`-*O|#RtI!tx5CD{mE0pg<4 zul3#X_DkxHIwV_fHwb&16Q+!);Dtz@!7H%DkkZKH;q6uIuy>bDwO95#n(m|Gl9SjF zmOV~9sW}`vZM#?^#vGBF+-&bnG@=4&gmsvQxt^=ZGd7kAlAuIof5RxW5kMB`u|8@-<02Hi(tP*{mZwT7#zxlW4!>2E=}VVG$*ZOk{R)N6ZkDTK%8`W9znxN$MR?Tqd86p(2 z%UG4t&%@>D!SA@27U?&Bb=&c6n+=n+51v{4%D;YAM^?3fUE_byu`HwGup8&J8RIGl zOc3%zuArqbQMB2OmKx&&1-uuP^Ubhb?bih$IN$8d55O@_#A0J%8Hii<*<_jCpLCFf z@CHfPxSPR32tlkmS4yDXjglMk$cC7^2-?yz8G#rjpaHm4$m0mlcWDfO9q3P@+eb|4 z$U#{dk_hWjoBT%`*YeM?&z$$!o9&H*T8ixmQ%pCc>UU$8$cpgWE>irNwNTy*nm8EADyMm&C$;Ru8Wm)gMv#QAmpANoJnq|GNaEDaT!Pj zZN&8sUX`?JS`3FWPx>WJFm7xs@968DA!PREuCbX(HEyM)|7tCeaU3EiCAHM+y@i< z36mjqF>+l?8;|eeHtPi%q*fvrd1J9%6hqzPeb^@*m8zizaTGk73$}~87?BnuiZa+S zO=E1?k#(4o&CoBF2NOM@SaztXfRtN1L2{B=Sh{hb4UWJl3eOz7O&8WB9O><`l}ptd z@+545xGk>o9JJ;HLr&yrl$5j(+ybgp!?nln!1J&?ipl_11qnGcx&;+TwTbVM%jgxe zbY%3pw&5CLu#J`8hRn1j>C(Byi{;23haD?Kg^duGSO%aArpqn7{2ClJmr~xmsBwO}XiK`XlM^0Na=gv|*-}Z`yPJT`;y#Yy6<@~W zMVrSZ7jeu_zPZx^XVCjb4Jz(zWwN46gL8uYt z*u#24$`FDKeR`s98%j>|QJYi!Ytmc%bdJCL7p-sW60zrwae znb2pOP$ZM`4M+lqD?v$j>g{0^p@6}h+%v&CxXO0y4=F#16=~8tO<|PpkncocM^H)n z?;QX?MQy;ps5dB((+EO-d%3L3HQ!SZrM5}ZWIb8h?qKb=9;f!__fqxkvL`D3*cDR% zx62O#U%1twu=;+QgT8WL`71@s|8hP&6$uY>Y@?C8k5bq*&H20;Z|NLHwzKQ<;q-e? zO@f8cNRp~&olWYu+YNtVx{V)3N~POF($kbTeFi^k&rHte9jf^~pc9GZ(*xk`y)>Aj z7qd9?PB$pLSKHDlUOED>e!-mxeYl9=#6goRT+Ug&j0|8iy5Gx?AGC)aa41Xoo10-oFo?x|{wR!)*qA++Po?ml zHS@lykrmG}Gwy>Kxv?`1%uY5iGSmHO@hVS``fjbU5&ue#a0zyWT$x>5Q4L8wsL~%6 zXl!%)w<;RFpt1SqnJ!l75+TE>B;(u`F-5?AgBG(71l7ZxE1o&WloCp{?SMHX3BSRT zF)0h6nYgQzlYG}D8L95gk4Qg$qo>SMYUue>qVHPx4EmQb@hnXmsgg<&pGpn~>rF@V z1soFBW4iHG=FO*P+4jmw#;lFp=|9FIbBhEqZ?~~4PPzm)fECYg##hnNiAINm(Gd1E zk+Kb@CoW409w=GBaXjKoiUK0vlgi>rQCt&)u!wcs;>THW4{rKq%qok-(n1{6EYjYS}~h zbSqqz?M#NcRx>DkF@yq0_&8dpkv29gZ+!F0PG$r7@UqoaOLGOuR&ZYOGRQ^p%y8{+ z&a{kS`?9}YQYsj(P+bDfYPC+@Ea0>*9y&8Zm* zHFIYT$PHkW6YnIjKXM+OwC5JDWpX9tIX9x0#>WY5sOXFQuxY?W)35+$scmx7ZpoTU zCayp9?|)c2bvJ0qA)Sl|)oEdQsZU`7X|K6w-W#2aLeA_9)~@gF@apwowjkGzfZDLR zk)daQ(edV^bJEdETR>gAoqvWsKK#bdq#tpCLtid0lzJbLfnnX_CdHXw0(WP++2MF_ zgPwE$hG&)pABr$8xmqXb4>$9-+&$W4eCa-4?~v$kerDNE0ogU(*By)(bliJvr+)H` z80I%S5IF1NR=8V;*X8oI=n0jM!T+$~d)hAH$8~L_A<))NT>r~BjWKlhjJxluy zP7K|0*>t!HvJZQtdo&yD`QHNq|My-%4KKuSe7Vn7dSYp(KFTG>G&5~9kKvJTi6lh5 z205}_zWEcYT7I`^{6F~fWb=)=-rLM8UCGuyUvJJsRD_^#7l&CG)8#vfF04n6S4VfH zj>FJbNFfnP@5d~ccD0!POWA;=HrcMHD_1%)nkn5 z*Rap7aQ|R*FVUf1A=0I#mkHlALX&zb_lFpBWz<$(x0V^Vk^UzhADa?CT9MDj_0ExRV{~1 zd8OE}pz@8bG);!0qxXS0pjQSDHzd2Uw3&ho?5 z?4eb)c`Z(();D2|W7hTI_G4+nVpNJhH<(qjjo3Q7DoE&Pm4#ouxa9Yw|4`M_*eba2 zhMMq(HlJ*>?z${}bd2AMOa#AMB$sWAmatX_NQOQjy*JXMJ?aEhJuz)}OHC@Uoa6Yq zNqu(hO&|a)Wn}b<>eEp@fGcSG?l;5S*1Qe^@*%wii-G%cM7SF>={;`VKOr)VFO= z7#7^;PTVEkSniaf9N7}ANQA+> z)$Sm;Y(GoIVA-F;R3`HJax(m<$$(&e~CB|Jg;-wGH8{Fq017$v}}B zaBv;06LyoDVLl#>ZCjsSqZP(wj)%J5RP@_S&*7dOw?%ZZj8U2%@z;nEw?baT8wFx0kY^TqYn+NL(L~@+ zBP}E+n3;XBdEN1Yd)uvBlN6(8cHK?Rz72h0SaToKjP{;z7LK zd<2oe_N?mXi*ydBT#9R(9}T{+@&5#x_AA~Jgw@0S?&Afo@>KJ3foaw@WD?=!c{l^~ zf`qEQ`)gyKM$Svt(zz~+uuI+kbxS5S8^SXiL zfrpC9VPaNSiR-VP08>PTn;M%H`+KK7#R$99WuVSu(5x zOC}rzx^f9+P@3k2Z=fN+dhFIWu(C%0FFM(9}XQm|BzmZqJG9KUf*_F_+%WGI_h_2y&3FOlmsL2l# zSd4VpuCSr44p0=04b0X-R`^{0(4k)q=R{czicTTX}Bd&&Gw)8ebzPlyG$HJzsjhZ)pf37cQ$6ujvg z+DvX14)<0Y$~)=7s`+PA#F$>kfybWl!_DM}cT8PFzV?mlg@F?Ttq~%6R^Vue(5FVj z0Q}l&bRZ*Bll9SzFoZOzZ&$8^e}VlUbt~KHCUy0`Hs}DEup3)x`gSayKY@p9}XaDruxwe zi%gnYkc?-ou(Q%ET4cdXot;5>%)IM@6223X_TtXxLbIzcL{YoW{a2)KxDq*E;<-}E=P@x~Mln>rx&?Cwiu~X} zNiLCGE8?S^2ziOX+_Sd{bB0z3Zb}Hwpp1pi4cuYE$*o~1M=?Vv`Z^o^ z`Ql}MJ$D-Y2fB*dC(oaR)1x%x!#EWox>AwhmSDp-qD?S%f6*V=8yfXR3ThJdUveYi z3+~AAi1^cUa$;Yi?e)+die!xiNX^U*(gz=5-0(=zcoba?ZD*q zR^I{(k5Y1|ixOWuIQG6x%xU`{cSX_wk9|2ngTK>VseP}DxpQWmvRd-WHqR>koWr@& z)-@+Lc4JR|uBw=c2NlVfiZbPsLItUQ3@r(NZNLMLgZUcCwNDN1e!g1XtJg;tkI40c z<&LCC#3XHa`^j+;XqJQoDh>dQXCySD%pV!;_vIVXNvuwytA&W;jA5TSP?&bwzpz~0 z+M)0zONO@c1JKW5dKWbE-{mHiHE#6<`!(L) zEN5_IjnS5e(oL$+Q+;xT9S+n6D#gYSY>rXu`|nPB5eN^qahws_)eav2AR%^=D_nQcCk7EEfPIGgUnLda>OffDhx~jO=F9^nqLotHeZG@)<Wl9&Mq`|=J)Ly^1z1zOg`b(1;^%{^7 z6CxiMt0a{2K2zH;S=`RKH?e^cyb$kUN}3VyzA|s=6y`IN9l|-Q-`09-WC$lo3K`XJ zP6`70exKh|nz;BsF`Wc{k1$wTm@FxhC!tUB@Yo6GPfLNgtz17;xqs)9-Kae0w}{i< z94PSY@T=9d#_L~R&TmT*ySEYQ`0d5N_-hPfKN~1Ih3

    L#E*NHP9e9^Dvg6S!uEt zw}NR=&)v89Ia2u+?iZ@K(68Fwlc}T{DL6*_s>_KJ3ug~sD!cX5_IANmUn;asQV(Ml zvXYzGW{4!^0^F#`Nj3NXxE2gkofnh{8FBv?@DBCItMMwrm|+Q69su6#y9VG3sXH#0 z;{q?9Y_s>WseSCH>&-SePbHGks(0I;N*e$1gqfo6oD0>yiLG`vL)H(gDfkbly0T+W z_S!$}Lu2oYClx|B^NdG9AbPt^8N$oyVCqH=L+^$HoIHk_H z9^SqVDxmEvC8u4-;J(_?DR>i!c3qx2Oj0#{kR;vUX)i~TRuw)tq}((k!>lbWI*{fU zVJOhL=#vaoQx+akr_3ufJuXC0{x|{`6E{`-jn5rtLs+-A55!eEkSPPK?gtH?J;4_T zY6rVhd_CTEnl@_H8U!=#_T33$)MwILL_d#LesBPLn68nJ;a6vi zA**T%$E8#(vL0eW7AwSfGEUrscVw8ZED6%j=I)VNgug^vchednaC z+Zsu5e0AL9BDU}d+^7bdEas1BZU^FTRmdRPh{+2SL)zBoj3z*estQ(+SR^wK#k;1N zC2>ekV9*!YH~Zxt2{{w6?L}h%iD3;4`i85h&4%W!0Dr;@#D;Qo`@?xDl<6du;Sezk-^@V$VhO)s@V{vwXYkP<0Ty>mLbl6fBmSHC98E2NUZBlle4kMp~DIPFe z(FCualnUHwnxdtP!TG$+rfM2rD}({$|2&}6LN5{8y5aQqvGHPPjwekMN)Bvp48ILZl}31?fSCC)ox z6sO0~y-@g8EHCJ2Hg&8KF?dT_STMEBug2mph9Qa~d%nNBo|;&sdRBGfHND;Gouiz1#f?A%-Q&8r>&AA{SUsW(3-&h>`ed}d`SL+Q8+NbyyWJMZ!H$Qt!1VYE153Vrp? zzYr?s;$AOA%I*D+1yE~$;S@jUYT4C-pS(|>@hs5T)J0nR^#tJg^yP}Or(`A}SHuJW zjQWkc`t6EJ&|V`CtLXKye0~hih;^}HPKF$ZWR5ES%r488L;5Uz`Sr|8?}e@|bY706 zwDk#-G;khgv$14QGY>XL#@OXQBH%Tyj1G7zlD1+^JovlB9hD+9eZ5=N2gvO`w&6Xf zMKl*(Yq6+A<2~!`ztk3TKAA7ASjn8^OeL#=F~v0lOk+)YBZJ+%JZ;HEu&%}Xx#MFw zO&14a-Rl0nle)3X9)13Oe1#72?hVPJJ8nmS?|on<>#?j@Ho-}%Gpp}kZFgP2uQlGW z;u_olu~1{n#l7{OKJXUK$4x|srKgUy9eD1Nf0+gq;}Xv*#RC`}$y~tAsexGcnVjdY zBoMXv>#lZc0+0V(A=WwY(H!n@5W0!IDW}h?u-r|jYXo<*nS67sxZJ>o#(LT_uJDLs zu%*%7mD0erpM&GeZSii#i1v{=ahMi1N%AtPJwn95YQVK~2$K?}{vlM&I;>AP*zrCn z)y6!uuVZcL_;G^2+;hBWXJGP9%mkSS_Gj@HTN6hvGP{n*R5oh}^-V^HHO_jj)2T(y zS4Nmiy6|;NI1dcFN4zl+*NagR8Fm?r4Q5b1{Y|!l9CGf^`8ETk5Z;i8|3+a(6mXP){S3G+J2cjzG#T`PGh9*Y<+Z^Ucy&Gj@MV>dsp1lWw zh7Fs83-+KI(f%gjrb+W0;%(8Y53H^~fcG4t2sfJd+`jmhL6LT$L(xVf9AN@W~JKA>MQmhlUrQQF|A*=j+pabnEA@w96 z*TJY;#?vRqUZD4^c%@ToIP&eg_wi(gkzUM1@}9RIgWnXo#ABh3RHDj5cI~0Gq=?B& zmuLFXEYcF^sT4m=(EcN`%wcUzoXb!U^bFO#Fiw2j4c_6Cm`@q)nYhc>u}mB;9mIr@ z35s-j8CxwRmlD1Ois~Y4${qBhHEiAdRh*ayF3KKgLd$S$>zIKpYKs;Nt_id?arsxx(WeN7OwLTcz;W0*HZlSa_6)x!T#~|t&(P^CVo=L2kzEr`4Sd(gl_Uei zfkAkt_h1~mumorZ-0}A#wfoIp^&ug=kUgPEy6TuogT(sp#84|f0z0_jvYn3B=){>r zL`=ShzUIlUbBHdJBhNIUv8T+n-7z@Gxpp83 z6SaWbr`l|QX{RuISTKP1(A9t?ZP`d#%hTKH&~@^~?R)6qX@JT|?S2M@qvrK$nr0n* z4vQj~S=O3)A0DQOzMo-rNMo=WAy^Hq8`w>2j=j3j zgi)xE!Sp{=`(A}S=m-tAHst`D{x9n=wCf_5xpRMdjw zFCyz`UJ)FuVmJ+~na{+TUzO!DHv3jO8w2=G-7HBsN%Ebh#=0Q~*EUYeBTCK0Sh%$tT;;tE0Q$wKH(id)Qo6(h0ygAHErG$dYJbm8{Du>}7eS zeoTz?Z`Rjq`}Tc4Ens~1HPq-nW&7H@w(H5w(EsVU=WerMS=#K6C(!-P2aP~<2Y6rh zF<>PEJoZ0g{kYEq&6B;Ig3bfp7e_y$t6HnN{pVFvdo4kA_LD-JV%`#OQZX9OuJ7Zk zQj)x9fq$xYgHR{=E=?W1jepkN!^9jICtTftOTU$l`vaM`Tp-`?D`fcoSoRbwK$(pp zg@nq4OTg&!bsqKX{O#x%syvPZb?nM`V_9tW>O) z#V`-G!7HMstHhJ^{I-Gy4o1@b+fd)Hxumc3FQ8S98>M8IR?q`bdSDPvejDxe6~J_pK!E`+d*eW@6>F1G7)Sf3orLC%8D4L_J~N z&Ak7JJ=ND`+Wh5h_3^y=vl%IS_}gDQXR$1(7bBfO(K)Hi{cfI9lpog#78cvdk@6IWd0ZhXd=dZ?7g9W&hGAKRZ2cXXS^ikIztfLV0C)@U5op8d!Y=fm(`N61YDO_}|kvECaUq1gbo3Gi_Lnl{DVTH;&O z%NjJBl;X|n8_tq<7x`Py>#4KeO|ax1>t#jZf&VQE8|{W-naqsW;$2#C`PmfMNv}8< zG}?nW|1!C%RM3>w;AiLqnteV0NHu!h44B@2*W2<~Q!4NPNgDG?!++DTwzt6V?U;O+ zd7#(j6&NdF1{JSpe2^Hu1=ha52JDA{P6J*yjXuB~t6e2Vk@h)vTQPV)FrUb^m^W#|B}qhL-P3OIp1)fk8evU_BrDV$O!A%T|LwYv zVSQa`nD8xcTz{i0)&akIVsagDFkJJsGd!W8_`!YoL;?FGJobW+>AfoVCoWp+37UM} zDbtunzJig5lBTen0G!|qSw%~#1qag$MJ*xXEhEdh((W6Q8LkCts#aOIgUxs$PgQv{ z@ls?wFVeWyZjXqbv>ie}Y@5us^R{F*vP;}tI7gB)9$_-|&5w#ZD2=AR@Ctilc?59Q zDk9%4X{?kR@xH9eD$3R+1`u0?`V#>LQ#po=krcZEVDXvBb}=8YP;f*w?;C+$A%IEc zkRZSLn?hRm6}o~Q7ylHSo-71|J`*Rr~9%z-gN0c6BoI-*+5=)zIY=rK4tk2clokjUi!TT-&xn&6))jYyMXjJ!wi@{7~R zFeq>OV#^Pv#nu510@wHPEXgH%jc9b5sy#wtxN#a*@CDU1#(#R{t;b)p79F9a-2CRz{AzY} z+t+b;?F=ldA$6ZM-;Dp7;ZbA?gkpI4rh)4Z#G*%I?S3EZz0ebV27v^Z9UpiVNJlmsYy$oI*B~JA7YJSchF7JAo=(od_Xqgz~ zF~_$g#C47+?TTa~1k%NO{f^#tGRJLi7|gTv{lZUy_y&POvrcTS;EUv>B_timwMD$F zKbClau^j)GLg;X-%UeS@7*(d~Fh7)7dzz_0GGO8UE}X2b#BP88`jYK%`sZbe0WraJ znOE?_NM#*kH6#2LFZOdxpv-Qn`)iZ5zK>17SG>h3(QJvcp>VTx{O>nWjB^MMIy!pn zLhD1QMLyaNJ{{do#o3ICDnZ{ z>%w(M1iRj?pix72g=8K%r;{RD4IuW@9^j3rw18U7K6Uut+HL~5EnQ~3gq^f`r<59? zIEa9A9J#)x0>1y%@H4dGcyIYzPv+q5G&DN0F>|`ZyB0}fPMJ-ejLGeiIbrIs*7vLj za7a!(-U0e0yQY!4^BPI<;ES9rw{@0>ikB``rp_;iYu(5R*rmT#qN_!CF*J#D}>_jW%x5)&5N%RQ>CgA zWy$1?zEH`#{*TRpt-Qtc#ijJtt*;Myscw!~^2Pqh zaSp+BP7WuZ#b`>%B0Td#DE!G)1Y0wvmc(znQE;?OpvGubOWeGxT>w#%M9huvfwckc0McR5u8kaTrdd1KLoa}ChJIdB^9ksZ}{l4LOzZKWWEiuMRP`2&tqC6dw z-;iyRRUmsR;{-FM=ROY-Y;jn1<+kdv$jmt;l=-3Nydl?0Wj3;gQ^i*klr-CQeABoN z#Ljk(O%j`S`Ft3o{+y3^ffLgvXR9CR)3ii9$OPSh8uJbPVZ<-q)IBz`G6*^&{nnH_ zQqD4!fsCV2$DK4|i&D3-Cvi}3IglV+&Jg%RX$Rpz(o-g0TFeupF~>z(93TCEbd>8- zib6Jju&y|n<#S1W&;ilsgFo>KbSlZN9&BF^@u2A!cF8{bATZ=V+Bg;lb z{9p(0Vw@3jj#{StfPa%LUX-TJ9HwEZN9cmk;MAUA;q2}wuXEHmfMO*#aNomq`(4;#lY3V!wU#V(GbyY-iNpW8BkOiS z!Rd9x378}LTG?ZAIbj8^^esS95S8dNZSSj5(32*M>F6t+O0ND#`h@hxb1)-EPwJA_ zH$!!HGol`Q4>R{pg37svLreLMNxia*o&rjXuy)mHAv!=N0`*u7VAo&}kIgCYnNEgu zixukz@_Z7~zh!BM!2wn1y?g@G7WeO=t_eWTn(PZtb!BT|!~$)(vyZ5=q#j?`7aG-{li~wefs1##Ag( zJZnrFb>L3ywK%X14z1Gs3b%Rw2T9fJA8``VmRyN|{kj9l7#t4D&$$ip886(N(SIw% z6lw^~ZMcNXsGjQHa%`Pw0$ddgg;x`~9}qYtMNXqeW;=K!tcW(h^_>g3I;mx%a)XjP(+pk`$=ve+RPg-r-%nP10Y&RGZWHG<(I(;;!ZDKPxB{7*=%$K}&(tAn& z-2i`lRAzh6gGE8_0k5r9ufiQBO(jt)kciYshE?R z@0wgN^!hx`*Nkx`uV- zUMepcR03E>?9~$pg@#_gtmX&3t?LNwO-y_Oqy-D^V6fjE=z(N9>(4 z56<60@eOo zeo-rj3(g28)56w6++Q;!7@Q@9hSNqSmlVHnu6`!-f&@a}3jgqJ=nFM+(~gS+2m_)o z7+kLH^jge&9aEM(jrMP3x!ZGQpwf{8ROe4)zjVy3qm%DGq{bP#il2yy)CiS zV9!~6^S&k*HDC8O6bqEVW-(B=v;t&Jsd~q&kZ-@h_Z>_LhU!CNW@dYox22O1=pAb) zl7km|g-oI)^vQ&Eu^Is((RMI*=T}y$BG<__OFYfx4j$=1<&~mj zTPG`~XxxiJKe3IfpoUOuSZ=!Yl(j#UdD{uv1ZEx~cRTq-wS`(gc2J&w?Hn9b8{>PFAmGv_@FAvi|E z4MjBxEX|bR64!Z$9LN;uLOU*IE&iHwj*zNVRjCz)lJ2=iun9d=u{niFbFz;WhW@0S z@%X2wzUf(4fYWBxaBVmt&h*8ac*Pbr3w^_h?wY84ewkUvrB~DE^clU?6NT2JnG=%a z=p|e7?CY>xluZ@Fm8yAX0N`=sRHJhJ$0AX?ukSpZ`xpwJ#aKm)@B>kFh{O98H12<< ztZY1ms($fWn4E#7M4Ku+S!cQe-L7aAPa(^!p4i~q_KTb}TV6k>X|A?OhKDtLjS-Ji zpxmU_RfNj6|1eY-sY9j@;6F_(HKNKem){u6KS97l%sujNhV5lQtId+78vnr^0HvnO}= zjvE>sb;dlQUB0GzPc3;JKD%iG>`S;N=Pk>Omh1U`FhWyeoa&fDeO2FDo_DO~=8=I; zMY>*^HXg5d(c8C>H^$3Eshi0jfhlS0WFl|8AVM8aPjjIEwcT+dSF+z7!~EeU_OjC# z!s>-&+8bTX%sYk7ZBXm?Ni@*4Q$dta`^AesKJLxWuTLnJBphbd(f;R7z-dtsk-tdX z)wI?_7HDnzZC~_b|9r6bvh{7*>G?V=k7N5zc>DG5duQ*gAa5#~|8+3vw)d8`_O13~ zwC4dU?-_}U9@PIA^qzOe?-VfcmIWH=eHi^%1tCMS+WKv-d_U3aIV5`{3OHqb@0ta@ zJ$&RnpU8lsM6cmK_Ce30z^H&5ESiJbkIIj;fFs^_z&exY8?DIgEGP;DxjCuV8O=%Z z=9QU^rk*WGk_f9d{l=iu1uFmEZB;X_wElHnkCQWo7$wF(gQrum@eRG?@sEUaOW~3@ zH~$}nQ7$-#r9K{yw)L7=J>>%N+25)BOVi9IW!*&G+INZHNqDYNfTLUCS$lsxw{X(8 z5M_9)g7%wN`E3Q^9an{HA@`r)J#&l%mm^V>^z$ClNbJ`7PNJ<=>0}ZLg;(0lfSElD zQxL>}vWLquw)Hs*GcE!RZ(J9in)(0y2q(i@u>yE=uLlOBZUUw>hzxB9d;K-385LWS zZcg4*J z*`{L?Qi7}+GI63J56l8b%78&mq2GL7CC%TGAGKm`HL4wteG?rUc|3XyF00PU4_d|5 zTRyHK%pSY-4^8H|9#xZdSZ;2o@u&8gQ_J>x?~mUHzJcjH#h>G^fcE&mrNp34dbJ<} ztH38U79o!nwahWW-o4;jzd@sianX<7Q{vaaR5-lj27My6^V@A>7QEc+0smn*k?Vq5 z&`X8_&~J0fOSfl_vFBEj>}5{l2=rD!)}TDRWjHA)0=g5qe<0HZwstoi*7bax_guM& zKDp&_{PkJs+~a-I{z&QF>Fhn<27(WkP}j>L*>j)MGfMBS_D7@d*n+&-!J8`x&F3qb~A+^1IUI*LLo-*Q$)Q$w*F`h-9Af#yB4G6cG4b8g5Qh zAR^5>+adsuDLsWZ z>;kA0#IrCDbF#DSY_@m=`>Ivu$<2gl<0L)qO2acKS#5%2wh>YLKNg_LEY&B$7G`aq z-xxywf^r4%&elIqgf%haRDkSVOw}36I(_3H7)Ldot{!YN77rQg_ummMT8Jk1fLE7_ z7lfv!TrB|P{>lm?Xq<+vO(_c}`O;xJ^fS2(fYVG$KdkU2Ib0Jz!kCkf7v5_+0aMD> zP`8-Q#Jp>Hs|~PZvLgXQb@?gQbI$;|{s-7Ui${WwAtpQ;jtPL4v-dYfDhipA5dVI? z8a@*VukHT)iSob7EP-8+98o}3-ZMtJUKkbd3DJ<`T`r4Xb&Ax$bsV7CBP~`kn8Zm{ zB8Nqh27kLRurJl)>pNW~MK_B*vO%#hLd%G9V^A@nqC*@wl`d@!ZBL1eF?6*)?_mAe zmfk#ZYna?F?_nroy}Y)Lujt%|RG&z2&+OgtCM3NaV&R0MIQCyM0v^bTKD zLTL2-L6|;FRj6#yokYnseHuJ2z71`jnEw$KTkzw!coCOsb4+-&G3(EZnaervnMgir z8|&%vm>}~ap@g`tA9I|}`&dCavI_mWZR@-F@wCC*K1TqyL8V*@ly9dMvssv5kOFnb z%z2D|77Wj^Is3_06c9-l7x44&3zKBIYTBA%&3(BoSd|Jh!>WN%lc9x*wQe}WG{p*h zJHs+Zon)re{6@l6%=W69dCsgW4m!YH(i&|>vRz>z6zHd{TIZ)0BRCAEPFJ9+%eGbv zhOG?lEl7k#(r|A2IM*%uTfQpuq&32ehQ|7RaQnXUaAR8uqpk3p$kpFcA(wH16l&mt z(FZdc+T_S=5PWTinckF)O)uMXW%Jvi%J;K<(TAIB;Pa@__XqBE*226$$$77C0axGr zkN>u8CX>C-lf8fZ1%0Ua-zJ^c8Ueo;0e$`d{`Ecek&6-9bEZ zvyN3Gc$E>$R%1ZbzX#pP5 zGWB42yk=vvwXLg>KE)T{+y?uE+-;OMcG%k3s-?$0Y7HgOs#KurMCriuataj8Fz70z z=aVr7N_4DHDAT*6Cg+en7h1qpsq8Aa`4K77o67&R>EkuYRQKs)pP}<1MyRjklOJNI zkj;XhAnuPQgj+;D2+EWlj{TlZvsf<9OkOTfoKVhh$L6_38o^miM|Jq-ekGHgApg7m zJNWv1#=eARPYjvY^(D1{j-^;YrJ~Hv0+a9Nq+JOdDg-K|cOjuUeAkh&H-0F~Fpj{%ul{vdK_qXiu{K4Uwb+9IEu*Z)=l9OO z?!npk(l?W;arW|Y<7!+3{%U@01j3gq9>AgfI<1Cy&PuE49}2p-J{%1X19lAr%mmpc90Bnrdjm$c#WCz2jmR zG|#9SqE`vDI!PA@=8U}SNoLP%LJtl6#>RiAOi_fdMQcnkQOOtRg~D{<@3O~TuL@jL zpRftBs)P3A7^zs7n;w)g6#xH#@P$a`XJS{NEgJUmcWf=vDUj?90%boZdONQeshh&S z@ZG@>R{xp9SRVb9+m{|)m{X=7_NlZ-Sn+$gfm@$<_#n~HA!iXw6|_M6F*XNON;L2g z!4&OQP!Z)u)aYXpXd|VR0i;myKyX!MZ^WtTX9tKO{^3z-lj*GGu? z-y+k9_G9iXE)bSr7A3ww-|k{~+0^p7{(M76DR|7+TNIjKKWp<8zw=;#;4=I9iR=Ha zBD%H${F*|2w$L$(*2;U55#HukhxNCLt6V0iP~kJ?Z>%dU^1}N5I+U}YfbHrri(JqJ^~o(M6VkQUz@O`JKzS#0j1 zS*3hA3Jlhg%2mA7nXH4B$5pQiO8U~2sybiR2X~L-G7}CJ0DXhd!@dk?eYkdIW}Zfn)pFIla?x|DU@6W+9EPP&Zv=CEC$JT%4O=MJi1&hn2E|sSX*5l}wW42(JIfa$WU z*`AVeJiYvP5&|j?&R|fSNSOLm5V8~})^vu;G3i^&cv+Wc1qz2z*aw>>Aq7&yML2mY ztrmE;2NT1p@@2)J{62AyV)SCT^2-xMS#V%` zr7$IWMd@Qc(}^<8#OoDRr-drgNI8Gm;9#Gf7!aTu3_^37Xno*7uxYOKRc`1{00$$B zj!kt1N1V6p>JRT^yhqN6O<&wbe?h70(WDHGV5l!q!XXM3qE_lc_BBLsp2m&Z%S zm*>J}Z-^HTeElmqMNFR1Sd8AGVDkDL?KY#Qn4Hslbi-FlL}+!%10}1#R~xEvVthhu zqrCvLOwpVuEAownM=nAMEId(GGQ;vBjn=^2*Ng~e=Nab1F-^Lq0Dn`Cex4DK^MFwR zde+!sBM{Dr8eWZFx;!<^S4?jLqi*pEuxPsl*sWe-ur2ia^Cw(73cRi!Lu2$BR7ejk z_P~UUN))&Y<<%UqhN!e?<6tsq{qhyIsJB*+5vkSw7aQoJ_5pos4L61f^3xf(suU;P1{#S>g zbgs8Wh7o;bSTG>eb|ke!4P~*b(7=Tq2fqy7#v8uf&HL5gLX-&D8w#Vx#0FN7eZk*I zz17o_Et#1gRl_A~h`%l7@Lm%lKBN}(TE%D5bC}0mocG3yCmwpn(x@FU@6%4K_1^3Iz-^_C~V5n*4PPKa;a-F_89LVU%2~;6X@9QdW0)6E)n@J@|lTIgSaIJvs zr$iJVi4<`V5c(X|Xsz#Cxg3E8!#NG7Vl(Cw17}=8Ja0NIm7Q~Ow3&2#3x>H=JQ_Jz zafl1u9<2j@oo2(UZ(hL_zj^EWyW^D0XJs`avtk+f&xIogZlYCQy3pAkyn>WZv&PXm zCIT!lwFzZ~q2z#xNm78OzH2ch3SdaNpNA(W32h#=$OP>d$EC4U9RpVlI9zaJGONTZ zpYY0f2yy{mz{{DrNd}J>A~(CwH(eA0q#`vA`a0$M)be67V;8F%rJC^fJf*+aVr#|F zu0~MEY9?m0Rj=pY5@Pg0KPpZN`j-8tsTM3d%mOr6osCi@Uimifmbiy3etjY~%?@)n z&SQq~i*0d0vfTu{h&V9dbSo0ko2rikaAD`6=q!R45e)}A8M+EzP3|}0D{d(ec@x$< z;XIAjr|PCbN)2LCJZ$X4pMXOyPIBO9ysr8^Pz(8)bs%s(dsjWj%1a?lzN*WUBrZsL z6w1TNXC`3*8QvzaN{DkejK9FdE^=^8TgJAS61~N(P^w1d&qf5( zyue)}=vxVVHW%bIZ$<`(w=o8&ro{o-c3B`7mlkliKDo&v4KjU+jzL!bRWdGCn1w#> z7pv4qR6?az5u>DUinVY1bF_knCt)2nWmR(zeCZ%>6~H@)pLc@c8dn7B_5;oI4?GBk zV$7(BOYk^_>O=(-$aI@|7N#*HtqF^7wA|Hpmi>_3(89O|c=s(nzC(rLzob+5YRYt_ zoimmq8{Qt|S5L&e>U$nwAeSP>dCJ%|0d1!b5W|V5z*|q!3+33;dehO z`XcjjO(wVf@d3IP)y;V7I^NvA-p#8`>nt@eGQV?Ut(8oG%Da;&3Jdxn=yew4w7*Sc zyVi$Z+z4OPq_XMdkwXXTPFUYkRyT6s-FRpiMBrVlNogS?j48&|u*mT_onnmF-Cb@< zsk5r-EUy7j9IvXDpE;Od>6)?tFeWPv&K+LiqVP!==2=jLxznXzt9GZ~uJKS}miItP z9Sp=pDxe9FSl5M-@!7x(dKq*)4o`BYqgr++K3`0a{HvPEP|BFeTxv41Pmmh(I1Ox%jZvYQGO;i&G0E^etmkjtX!?co&mA>?&~kvinQ=@rG!M- z3L6vWWj2{#Ufhkzs~ZS_IB5g+5B8O0bYd+|i<`Q*_Kg@XUfjtfnYH zXl9gen?zwq`-dnwjHHynI=eO$IY|S1`GF&I!83K`;LS!jcF%|SUbnDKo#~PwN{RyN z=3=ZmLF{8)&tACfR=8W%T_D6~iIKz7keyZ=N$8BAxFCe<$cf4~0{g(t(KMd;9ZIDF z2wmJ?4Ol-IuRF?pvR|H3s_=z%%?vBzqyE~8qIntOBTp{+6@HfPN!i(UKdC^$8$Zhr z?54dk@8_Je)cnOYT$=1OO29I0iZ#3%B0@1-ieDCuc{xO%q`!xKsMF=0OI(J@^qWsQ z#r0muJ$*pKafv(w3+<(i9%hz?Z7`8>860@l2va1;=nH+@k2zJ6@cDJ>L+R2Xp$c@6 z7K;k|`yjPMI z0+rCDm+b&nzhx* zn@02oE8yr)&*h&JOKEstU-hl09#)-oxtF|*2yp8{Xos)MwX7`J^moqBfi`yzQzCl%Plq>~q}`={oD5fvDa@%qMIpGg^- zYXX<%EtL7(ec~&VCuOB-78WxULE}-J*H=Ctm?+|-aVYpFpQ?y{@Bh*ia@F1b4h#yP z@p55HZCOWjWi^e2?iDTDha3@1;@q-x-rJWAHNSVDj*Tv55HeQWE%^lC*@c(!2aJ`< z;<2o^v7Mhtbjx*zE{5}yYE8NrYKKWk{gHG|#ZsSc`+sD;Q+OnS+BG_{ZENCWV%xUu zbS9iw9oxyow(W^06Wg|}j(xKC{=chp(S6Z%Rb5q2z3W{HggVR-lCyD%#U?)U>f1M4 zMDwi6Di((300k^S0G?I&>E2hTfG z5(Q1xRx{g)$w(Lhr}Wm7Jzv^gY}I|WBfMY7#C4Gn{5!g`Y#GQyC2%D^w*L~mnK_B- z@MPI;07Niij6Uk8K#4=ZQx5f*XG)=AD0E<$H9wrAVS>K>U+@9wYwUql?u;VpDJP*Z&mtz3Mq`LgMWTMr_`eI&K>Q!dB1F7cyQ=!V+G;{MQr?zZd7b87VMDem3l z{efEOq5f-Gp?2jl#AoFx6B3IpsQub3fvokeNi%iA8uolP+<06?dULisb2h1`9?P!- zjp{+wK>xRK4GLubr8Vae70=;!s;m7=XGD|ms<|EElEsASF=ZQ%gRngu$628)xk*nD zyYPgz1K#6B8zV4S39-NPLL=XxIh_S?&Hqc#P*p~mBdoVK^37a)v_W7Ss)2A%LvL!B zi=nQWov!oCWmLd8mhd<6l(QL?4tqC+bmU(UWtm_MrLi{UfbL9bw4kSDJepC0UA>!p z+jbO^TpF(tOZC(5U4#}K7K^6V@}aza=0>i(K(i4|jEwsHbqZ-`$m(1)j>GUgOh|g` z0%@#yD?!onf)byOkc?RIBPMs3BSLohb7%W(UOnYvknycfA!9M ze>J?&2z2>j42d*}TkaPG1`iytTJ{?l?!*=(lxn9N{qT>un+dBB4B^qk)iGgDKbZUP zUb;PkuR#Z_7OvbCcKl{kv0g#k-dG(h^@Rk!zIcI&SSmb2!uw_JRrE|mdG&B zWV&i&C^?`hR#^geT(qnfXB&{@zzGs zDM^}J;*fycuNMnSXC{vGE}7g6x@GTVY*x(hflbBEPcSuE9Of8*XvBER>s|uLDdBi9 zNhZh~YCHyv$ve9g*=;;nR^F*gQB5^gRrp)+p5I6R2(79@M4Yyd1WlTGWj|m59V4HJ zmC_$W{1E$HeL{bLT-Rbw=RIzmEY`Fi6OgJz`+4ur* zzk^7}%o+SK~K(B7T`{Qd4vT5AQnL>7CJ8>F3mZ!p?3ot|&Z=Q#o|)jsU5-V4IQ!iq!q zR{^mxjFZgfIE7mCA=66lcb~6fLU&?9+=4<+cem7j`%=fQbOo0of;SOLRpU54S3dXM zOizLC)g81-W7I-7GVKk&p7}e-r#^1qtej+X0S2u7{H$7$odfryjhxii+eeg7Z#+z! z^tK|Nq5@9Tf=64Yzi<~R*^~_Q$93T)8A?;=GpAfO8LZM<)qF+Y0nb92V40)Xa zOzuw=R4Y}Gmb5d8h~^k2#oq}d4n2QGS^lJvf-K)JgcIQnB*lT8h(v6*k^t+y;PJ#y z9=Oit>g2u-#eCAMBNV199JKfjLM#KrLG0m1fR~ODr2T<${m+(fXMlN^hKDLLAI{Q? zy?0o|(watvP^a%BQEJ0UadZrkg319t2&Qr7d+1mhm+^O0<^NX1PN@Aw2}70ZLlHKt zh@}X-6U7+{pm)a?pnU?~vL!vC_oQS6s~T3&Z)!#JlC7NGaj#^`e(s4|Um*x+|Q z1J7IfU-4^(MvE=Bi^*~Vf#I|qT)3c-r9#>REOOda<-@PbH-I_Jh?kk_5&pZ|SuDwt zOF634vN8l#)d0ywP*TivIE>NyF%qK}pD^JH@D~`sP+N%V84^F}5i^rz;$E7KHQ2oR z*ZV<~hnGOn)=d-=6wo>j(@5Hkm#ifCE8<@)&NUitLvZ2ubqiRDV%m~uSv(cl=eAK8 zrlMBm9W4YI^#duxF0ch2aPy$H2<|;xP*}77Cm6mpTG^MNY99tX_ZD!^L+01V!}r{7 zhkMzMAv|;BVISQ&6HoseY}p?#H09hg^tdPx?qR@=X+ETJ3^(&h-xgGsRQ^|yx}Qpr zFjLwtNY+;L88x#km5GOQ#h>({3~djyZ__zaO?5X(eh)JQPd`$5ST0&Q6Bz3#QL+?; z#`u%F6Hk0?o;S_z02Wz5(*S4Ot0?mxvIsZcHEH5AkJ4W#pR=!?0oF9WJKuCqq}p2C zksvQMsy*W0so_#oW$NDjlE#6=3x=16OaK{hseel$f*<_ZrEy>AQ^QEg`ykus4XbB= z>rJeGMx22mUFVwvAN||xS`OVvb}azvqnFXXCCt8{f$#+|T;BPt1*~>rJFuKLC7rfI7TZ<f?82#Z56KDI9MlfW&HDSeorM29;0jcGCafm z`J3H-ce8j%yApcDJ``m+Jh~?^@G&?K&m4t$%g+@`D0~+398=77&gLY7XxJG?A?Q5K z3SM@4gh*Hr90mCyYsOP|=6@=t4jCz(#oU$Oe+x88^d58s5n`|vfPM> z{(T?fZ=5I3OYz^2{}g5IzpXT;&a#u0deArN2hos8s*1&Vf1&KW);=Xrs-q=CvswHg&FE*7t|x0%1?Dz!1##+&_>n$v!?BfTJ+ zhs~OflxAYZ!j4F2^h+LTgJzaHT*fwXbKBT<^gqB~I&O%HmoPKQDMq=IV>_JcjE#&F zZ8&V1<(OEUR&;_Uk+6iP9qXr|+04*A!;IVF>7Nvd^4gNgssZo>_+_*_M9|C^z(Lbi z9CyR_PP~>n5i? z_^ol(j!(TE5$R!iY^|R{!=o~8APa84g$1=PxJ!TGCyVFlhDCLeq z#eloi*@GD%w7*auufYWoAN}mdI>Y}K0||_^v5{68_FndcPq)e8K@hhnXPb&xP-*gO zs#z?Ux3ubW^0JwsSFI{;Ms19=f-^tHk&Cy2$kKY~@Z5vm6{`&IaOI+`>Y_k1I--7> zh3=+xg^{Nm$9rHdhDfvD({?hFP(I{2p00>J<_Tn%VxD}Xordm}zJ(?7C`QgKu+&Ua zxm~5O7DSi8??k1|nd%@-WSt+61nI*fHeNIDLbAYD>J}(aTk%SCexr!Mt-oMCxmY*; z9*#y?v~2$`e%ZJms?dZQ(gxm;X@a?cw_#rNzf#@_?Zb2eanFKdlz{*FGaRQfv(P<5 zTR<$n0p39#rS);pBunI4wKet7X7NKIDsmI9U$j& z8;?0rUI(d${}%fu={(l2orrI(IXb^Orb|7L%nHvhc^%*9oJYPsNywpP($MMNjI@s= zzf8Y-Sp*fqqyctsD;_dfHKLv6*QHT{h{HC7HWT-*QYcZ#m7TZJ!QO%IB7|xJA_YsEZU% zk9k~o&<^p2o}A}LS^s|&6$#){5H~a=an&;=?!P@aWJmwysAQOPzOdJSD604rZi$pi zwOEDU{Cy0Zv{=!~U-MzUB&8zSU@vMW?0!c`@G_>ECPt~Iy6LrilJfuU0-E5}O`BAW zRB8Z%Qh(KPELLsoFEYdd!>Cr>1(g1lXfetoCKXUb`UhIZ zo})QC0O-YdJ@)J~RREkKc@eAadRSFEym0V)?fIyEKf3}B@hL|lBdscMDLMjzThoP?AP|$csPEZ(;bXqPX zP!G36&^fFyCrVKqm4k|I#BOtT-Xqu>qdR%L_akT^!MZly?_sTr?1JBHulLz7w&5DY zSY>U=6H3!sW9T#FRK_c5ZXs9km7p1*!zg>mhNTzjDa14ke_8T@w8FcaRtc_+0~$w|EIGY#~TrigO(G}XygYcT>yu?HC_h2qOX^uWTEp9=Y6@$o4vXJs-E zj9?rimz?>Wm@$ou;TgJe;?>dp&>WFv9F$X0;xuGXDG&-pX<~CO{T7`~Y~Lv0DJBWB z7X2aFevT@0;m+`K(o0@CWS8>KvPr74sNSQPVM;Cj7oovLy(In~Na%o@kW3-KaBvEg zS2;Z!KEB0Ud~Nv5-zOkrs|RYJ*V-h+Ts1(q0mpHBxdyedcdINEAe|vilmFu<7aZ)z zr;iw=`*puFG8Na(UuJIDV>!V!tXV_mCKmlmIe$(}c4l05*keqE_L@!y0JQpPgPHhO z>KPRbFU1Pj6>Tg;A?f_ZC$Ud08fU8*zdK&5J$JhYq7l03TQ*9d2aE@^UZJHsyJ)5d z^PBsp{X`u%XQ5zijMYi3=l;a+K>iPm>&lZJE;uuFt~L7KFVdI@@y5jjRzU+NHhp;Xd10{jZUdpu5n*q zK73`Gn6wO;1GcA5kjd@!|0k3r3>61xFL#rc$_)R}z;wUFcnI9eSXvuk2#EBRNO15z zHtv=Du=XZEEtlrdfq9O^b4@W`+{HuIRnIbE)Y^omfEbD}MA)i($FWCiNPEpQ^Xk%3 zz}7F7{`z81jnz(0(;(JMzif^Kap!qUo@)8mp zypyMXN=^kCZr#OfJyBl0qTf#zzQsCx68Sw$ec)+1(r!I{9mwu^bys8PdFFa=^gB`W zqcwQT{ycRDzBEs5Ckwr8eWHA>_e+JE;iX|Do)W2_pXvDSgy4=IFQ%21mnHST z-ku3v_ST{wQS|)%sYqT5>e)|}srtZsg?$98js$`)+w<(4>pjTC5Or6jtGv`eiXtB7 zkjgNs#`61z`4nM({t%~nx2A8|S&Q3;*BFX(aIr9}Mdbs=7>_tL(zm1E$0cDa&2O2} zC5a#hUP~06!~Ax8_691~>Td-IBDv0+iWN+<306k`c*aIj++EK-g7$?Hxtnc-&RWB4 z_@z}87mGq(fN=^oTiIX&4L`(ts?WZ(b`#F*jFr+Izn;w~P&^q7Lh;tV8GZ_VUEUP> z(IKcLrgxi6QyolGkfGje~kfeY`wE{6N8+}Ti_rjW5z6L>ON+;v&}QRw^(MzwT$b&xS*@v zfLDiIy8cy=(F~bZ81htvTebU#@;h&xr0>jkGbESQx?qmnzZu{fd*u6Y=gcOjK#|12 z6)5)dp@i?NYmf?;GfDnbtyZeVz3s8&(nNtk=-v_1cphttAZmP%ggGie@wNcD^{EP9 z(YM-OUTbufLP0(yTt>!751D?Oc^dH%Acmy6fbic+=be$Oszz2R_$nyiYHXou^sr=? z8RRK-G^-(h@9}?B{hOV|e{8{)>Ru?x_qei9r_a{{9@4)L*f#dFj;k=nCF2nh8!kn= z9KaIb1M?0E$SP9qNsGT=^P{{x-*_22yX35JCPU5eP?eH}q~Jzy5)D)vWd7CnKp%~S<;P402c8BWVw+||mx$+b{*gof#;WG9P zstiWk@k|mz?wRb1yh}|rMYf%SwEBe>CK4gDerF_VraV|v3L!`;l3Y;@^P^O7T2acmDoR7PhzQ~evO7b zkp-0YzJk(j1FYYuoN{7t&W3w2W*8cSu{_77g8A`j9mrvncu`n#7~+_oF~q!%JKe%u zG5lo`wCs>X@Et>10Aukm1WIXAvCD*jGICyCa6LIWx34E)G&1lYUPrMG^^_qP z90^<0m~5YnVH!z}=6DYJg=jJ3z_>5aLUor~Tdf}lKtYJ^8o>rRfWv&?Pm8vj#!?*p zb1>nLUII%+Op0Kf?a=G6Ehp;trWz#3Iolo)cwUF)r^S{vo-t z^7{U?mHaP|aU@i{$xo2Jjm3gr=)_Ls`8V7PW!nRaZ=};C>Bh7al^?~VAB|!mVBK9J zx@nU!`hQQtxP>_p*^fUYf7Z$3i}Qqv{&~xfqpkeBXG>`d0A9xo#&w2Bo@10W;s1D+ z50wrV!OaZa&w%>rmb4G?HMcNyM{DP?__*NW7`lV^cd{dJdkycv7!Z)`#lm1sL2C#G z00TW@K$)?~|H)vVF(E@OH4PQR)(F%1wfkjt*{Q3_7~sY&`$}NHm7R-C@J1ot4`b-6 zNg@k5)^9HwhmYapDFS_sN8R^m{v+w`@afRsc|A1?b|F>Ed}I6SUIzyL8@?`ljH-a7 zJwywl>ls<-WrLi8$RQ)B!h9C3+b_f0p!00Ti^b%RgA`mMC)GKcoPCCY&w+7LyhrgI zJnfwFVy=Z2d=56j!b}y5Y49d{c*`>0Xt_lZv*xmK{*wA^eAZ3K_X>bXJX;1HM;WATC{yu>vLVe;5xiFXKckDXpjXZ|<+xB$Gu0_=@*6+mAKbyxh3(sR_v`sLNy^md^}%i*gi$a*Lu&5F3p$g~Gp!hk-&xNS%+C+| z(Zm6b>JEx0=#f)XL+^`)Pmo^sWaz}w$Hr&>(uYBhuU>`>>{TJD67sF?RosV~-{;)W z!3ZF=r!oiKK7M{NKRO^@T>|ZoSTA0TV>vb@!QH+fvH%o*wT4240z+s3f zO(h7a@lL#IA_`h@Xn|$UxY%@`gcp!Xj#EUPYpYkE)E(#p$b9FTP~!5pS{@hwDBT40n}RVReNOB-iK`iUD-yM(j8LF?x?w<=?}#8+r{Mjq_Ua3U8$)ctZsz zwv(&FWSzHF{AJsI3H1)$ri}H>(HS>gPM2e;MBasf0Jkk!Mk#G(>hjwRX^hR( zBbj$L#^@#;e7Auq($k#ObN30nqxL(F%hmr@VQR56q@4v9`rd)OY5`y5 zMi6J8#o+~-Sb$;yx;%d=n-fY&+AU-*xD%XGm3s5e1^?8$0k~3nEFN9{c!1gOpwelc0v_?IHiZ@e zKO6eh=B#((ZrQ1FO2UgiYdF4kyCxTsGbwG$f0$!tEZJ7gD}ynHqF(A@!9@F8qq(BL zzGMh%u6ck-ZFN1-mUrA}#ce2B`{K=+BPoH%m3t4`l@g>(GQH*%l8sW90)#Of?El*Z zd}$25J#SNgnqQd>rReFZ^_-DcsVMhS;(WScJcF;=n=_PJxPcOKY7jI#KyTqJ*rY^D zTy>|s_yOxk#Ku@VJF5lFDepNvqM5!n8r>0ZF+V(uvK_R&>!0<8NeVYG=pS51AheZ^ zi-LH)X*6;8l@2-*T=!;Nqy-me7sGj$DQtLV3Z#Z}=Q$Pu;~^_Dp9a{}vSxt)BpK+lF1)j8uQ_R3n-?lx4u3XE@~-a>WJ z!8f_czsIy>fBH~bJlpwjSumn>cIG>IDjldA7940`;X73Ezcp-qz%0p7xPH;iZ(|1F zZ=y>*aiEo{?iZNP-rBVZa-qAkkA+9Sx!QSw-wzv@s?TOll~)b=Pb9BMpZ*L?Mez(w z#r$1jg7hw)G6xL;dk$s84HM3u0v5DN+IOp>Qqgmt{38oS=_mT0Dvu18-i%XrJY6jf zo%tZ0c9UMz^(y)S*P6kp*+c7>;%lr6TlJ)8FCXoWo)FOw?UAq=tgSm|oNsZ~e~jAY z6{&7$zwav~mYt->WGd2~3Vcg}P7JrP7P#&gvAJ1uuz<7My&b;13nOES)}<6X!|KB7 zHjTpAWee_Zg9$zSO4iL!?^*uqC`Xdj#}D7zeRyacwb5j-l(JsqL(xTC8-uJ(ur9+b zwEMukdR3R0Aun)RIP>4w2%Iw8YQK#V@JE3O_^Mft$C9Ld{J5&>C64kv%)(#sMh*h& zb_A}I4h|$;5pNq(C>vgO>2%(TAM;M5)_JvE_UGC9QhWG1!;Gn*KRE@g8B~@&7|%2{ zA-fsv$a=3DgWmfARD4iJ?5WN5Q}s4jd_P)7rE(!232j`qz~W99A?}Gc`PyRMGE?6Z zG#DXsvue#fn$KDmHf8pUTxr zRy>i6vG>;0@z4!bWK}Yr7qT$dao*p3pu7+HTDGh{pOa28u@Jf0{yX}vgr{Nw``!Rs zCv3)L=OTAs6oujGIqh;Zsic5mc!KNfUrIa=A`5ZYO8L*s)|8u^m4_~FIMdqhR(FywwpALt1*bWIADWYWtHx8#H31EVQ6!F?YVh-BFG&UU~~L@~2N^b@t<_=wecP>{!3^qiv@OYM<#o?NsDM6MKn7 z9v?ydnEi718z(#;-Ioy!MuNx;Ip0pN!kHmr`9&<{)t{3{)OHsh6+FlTqf=Df=uC#F99c^3(ChRBvGML1(G1N>dp{3PyFvv~ER+bbWle278LI6fU$g2*?L$0knSKki> zq1&y%d_^koC>8+o3`t+X5$a{j5&6+Ma+l=5CT5iE_RM}d$^yRI5dDy6hGx=RY2Md=KK9W3eKg+})ZoKnl%Ue-v zskWkHO5m}Z3u2U!nB#s@pp;kLb)mLHsfVCT^blEm+){wW|HJKN3v?M^Q0=z;LPtr%1# zy_3Qbjr;>Xx9a0Wq&OfVd*2L~x*W${Cg~CFUO*P&gME8@4Q!I;kcrFDzVlx*nyFV47&RVjz)A#z0+dOUx za5}e>3S)Uq!!BC49K*oY><1!~-Pb*jh`zqY7+ZlSO>8}n@0A9(k~hKVn8ryj08FLh z>ZsP!#)nuP&0K7s>!2RTSvcyZ!%d%7gU^H9kDa1KqdKSi)A>Xw!JGxE3eSm_K}6>| z(Oh1=IJ$>EX@`1^oWij0SzK{B+wF~WY)4azf!oe{rPFM_Jux*J02KH8WuQ*u{zRg7 zooa4Ey+v>r6>dIJo7oSQe~YD39bS;pz8rY@prA@Oaq!dAK>_86i6joC1<9)#7xe7|)%@OP(Tt*fgB?;h`UuRk_BllDDEKC9EVS z!yk>AmRKA=s<}PKb58mc;#}1|Ro_dhyf|~E-X$oEQ*t{onN`F{OEq32kM4Gpk*)7= zyleRWxkzN*b2%;Tq+J}Oca3J0CTP%Ba*&3`>yG2m%W~rc5VG@v?yghrxt%(od;2}D z<&EepB^lg(PyHLAGya}a3M7I$T|;!NrHBqSFFR{0xrtNkpPQ^ye^1O~XCtB1B0At2 z5)UIe_)2%nNOX%8zq?wB&`-xB38wKpKlV&3n$q_B#9m%0hoCeNe{}fR?bz}>h-uwJZJ~U3y{49Nr9fuiEf%u3REB8B zzXptfPXMONvNc-jsBPaF72MgtHVIn2C$ScnLMQi#LCslBaZu0$Zd~S1lan}2?~$#J zvfY4VnSO@{OhT@=6P#y%E-EU!A8(X^FLSx?=NTL}3UCzk>bf7*7zFctPOi;8F9`?^ z+Io>Z0P`m^(S{2vbk+BII1vL;R6Bo|wibkqA zEqz{AE6*N$A2R+_w%cFSp$$)ZsAbG1<*?4zi}ET#SrO7P3c4m~h%sb)?-w+f`2f_D zE~mFNzg4cMaATdb zBQYhOByGtlF+5H~i*b>M``~*5`jXdf)RV2oluU z-E5I6&~We>5X@XCeRtfj`Z|B{d0RYilyv>=fQuK716bTGkzvnqRFx8%p|a^;@n%ii znt~_6M>)Q$fGyA=m+Mst;;}!(!)K4*QwPa&?dP)fShcCH)t+b~ni}P@C2~BTSk_K( zLlX`N-uTZq`Ut3W+*KMUn>17kZ+aV3IR7y4-Q?kXgp3y^qmS^j^aB*8CC%?=t@lU6 zK~Ajvn4A82!$r@8HLxa;BG|+S!;*mRO&}N;e|lv8nsnDfOc0-B)nJ z@eT@nqR^m$aSSOJbw4*UdbJywAK(NO&S~ntk8qY0lkv?4&fAIf@n~RLG-ouDirkl* zi?m>?7Zxn=Qi+&yv@}SfKsnhS5$8n3h_-eWv5~TBuX%c_G@yw}jq(E!y^9qiQ2@x3 zuHKXcA_F`cxz>e<=aie^0Zy`Cf45p7aPUir=nMY`jrseae5z`oAp+_%<>WG90&DlF zRMPMLaFX!hs-Unewf#c{jn_r%`7E=;;Or{j{vDYf@sGUl1O}A&QPF*I!B)}f$EU4= zZp5!5v6vT$#Y}bV_thCw8qVy?KUDMK)nuL6Ze4!m%MKdd`BCLb?Mk*AC zU93r!Z*0yS{V0i7pzrU)D&7n>k{_5L;Rn`7!5ST|au0BdMCnb&q7VZOVmd1;zEDh| zB`}4EID2+!EMm*=%RH&=iS71|@e)K9atH%a(^YAAu@n{gpPmY{kRmy6$gD>J&X6wy zV#1+M+g&Pj0WUL98pvLZfBQ+lQ=n;BmW1azreq(`h9fEp{Yov37n_XiPPhi9lveP_ z)%?jiApdPi9}1Q_qO2*>Vv#UnRIFcet6Whv#-r#Qk2Exkz;Q=!Y;OK<#nGg??~(YS zddJOm#*q&LW(-{%`o4W1tEiEGpLY#haDJ;kKQIaWN8eR?nz;W zk-E7Jvu!t!dh|MHq??Pg%P0%wkhSGi+-i+d!&I-lDTdeYC?y=`{;W(__{J?74MCbF%eviVppH=Jk9!lT&y%IZe2^9|tu-n= zYyB_&LQI^>mrhuE$@u-p6ap=FLSoc&jf9SU2%09DNRLkW_??)nN%$0YahiVCFOS%6 zRb=p$6PWbK@Gp9jZN8MXK%9DP2yDBs#&m5xU=X2XjwL`u6e(QGqEEXLe$($UMdd6E51ZND0g_O)NFvFYu~42Bc-y zr9klvOeK+(D(5u*Nou4S)4@&9pZph|1f8{ygOQDTtGBYxM9;R*(fV82Frm?@%%Ur2 z>FmX)npclZ1p$cA*al4(xEYkgPyp519xfd;&WobMGldh$+wk_!TtMLG23A>@ID82WC`7*0XD^&;9Gxt37^h%bNX~-|Mwd&$R?o$78oQo+Tf2 zdU07bXxJ@;n8Uqzzq2Cu+l_l=y!^L|S;OKu;GJ1qjmNobt>^t9ft|~x=h5f=)&~Tb z{V_o5H{#IVLmIeA#V_dnjatB9Ih%(;9-1b>##+lRJJTjP-H;XEeKpx5u93iNos{=l zpo${ypmNn3P#fEBb!N(93_IIVd@5D89y{8JNx&@{CoDYrl7&G#j*Y#*VIZ(yAFc%7 zj`y{+l4>z(^dj;&oqYESmJCfh_Bthjr!03Vrb-FW5lP__O(_1~ndY~;7*&2aR>w8A z_&>N?hcd}JPJ5i&`K&vb38OQ0ArRac-6Yd)Dc1VtPx`espJ?5C)K}q6^2jsCW<(A} z6g7dyuh3F0of^4Au^!qn*SCIR?T5h>;|bd884dn0sVOAyXZd%k>zKfGETlubeZ~D4 zv(%^Wb3&GFF^zdFLm<@49k>OnAVU^ju6rV#Tfs;^=>E>=U1AE+Lk?$RG$lbnQ=h`F z#=iSQP5XT0XZD0DS4rKvUtM=Zbarrk`6=DJ#qT~P@9@bx2La-27s*d5RyC7|wOmGp z4ckRRZ{RVvU9>c{2Cq*#4%}bz#PwgV2C*+;jjMEOFEyCZZC#V)`V?C)mt3sXc0Ixt zZf=})z@MXa*fG;@yz^MK?A|{v2J#~hlvOeUJe367CU$KQGHL_>hi+0@0s&w~2 z_LPeE-)^xsRex$UQ%fjxT(+O3m-^WEA#u5o)arJVRmJ&4g><2=2o0HWIISalMqT7~2p=$s>7 z;D0iqoEuEBz)#2J=e~ML7q7PqMK3j%h?N)Gn=Siv+ZwbwlqU1(t$I2zB_;{su4Qm* z(Q=l%bH!-oRzUH|^I-=-Tcx@$&u9|NMe&|ntS-2EZ@t{v{4I-&t?+g2pBUi%&mH0W zMk^V#`?3VY&bw*+^JIQi^OJUjh+c~Qf&5nhAA2QH$`-FmE02WPrlD`sl7iV;?vSci zIi8a9f-{|SjO~FVL+x=d(V=bGA(`2O?lDisV5zJPZ|Fql@s!x#V2haNU)Wjg^yEuJ zcUiFuo5U=oj4p`5G1V0gJd55RQ2?w9ZRa`aJ0)L75uPo0}ycN9G z>v;@e%5i&ct>M1vv>&(PgOgMg1YG4?-$^ zLqpBg(MjD1)Ct^cDXt!uiM4ZhxUiU*>{LN_N+KmJL>W$kdA3aChEn>e^7_Cf;~Pn6 zy*X+A=Ub#`JTJy*f>fo*DRg=`IJBar(G+fwjx~YugQ4t+0Qgb50rBwSnae5mU$Q_; zobpokz}_aR`?B*d&!C<)u__22{{U;BK<74ie+2U)*2&Tx9vj^Y%yDjD%J=cs#Xh_+UirgP z=GiwkL@G!G>+fV{93ZwBG))L5@_`0abq1!DO1tlDt8(_K_CJ12l0y_uF;9xYBVS$& zPT1qb3TR#g_l;$1DS|8}CFIfTACqKw*kAO_t!6uq8;-Y-$WAj9rK z%t8N%%jcN-j^zLubVpuFvxgXxcs1hfOOt3wMmH`WwQiy~F)$)VV{!crPajx#ah|6o zx(huA57SQ`$K@OY48`y795E%$9`5^l3{9ZoLtb%`fSFY} z5^+0#c1sCJThHRR+l>|G%rIu^PNAnBJU-M5+&G@h~3Rx~kg z=j0&Yo5ahk5Hz#EtN~7d>#(_V`mp+xg)b#p?2E_1xGL>}@_WpjE1kKRpI=Q+CDn>G zu1e2Bof8w5SMwkI@`K+|w+(}VDGHkh0%W;>1N~i--}k|hK&o>0Cpjy7xQq0pLs4gO z3u(3p6o(iXHSFMrZ)4j_>FF4sS2(JuOU`s`9+Lgp3V6P&B`F*>B%h;a-r34LAd;z~ z!&i0l^QFtpG5ieNuzLI2M#F4NDSFv-ZAC|2^?72*x0Oy-oK+gw%eS{7Sn4AKOb#&TQA7^dH10R+--lge+Z>A=mf^;e+6MN zeUzu72nu{W0Z2a20U}6Go8FhNM<_mNSFH{5$ps}7O!&wi3^F*P~${vH8WwUYv z0tOvW0gk}ub7@w?ts>t=-QGhK_K(iJ$qxITi-+43#@F?u?x(*&D3T1Y_rODwuANV2In_C)fO3Htusj&X1Vd_EjVQu+j0J^kd|>uCr*b>li(sen}$( zIk#NqCVX$+uX??Ge@&nWJ(nwWWc1u#^nBweT3J~U`bZSgYjm@#VyU8FuA+cNF8MHo zH{YP#;8?}0gkr8j2-2r7H=N@B`3+Av7e8-kdUW;dRHhB(I}?XQc~%=p+pJ5!nH}q_ z$)Wg!H12$_la1GYDdljH@W#r>CU(J-2)pSN!--L^u`h5BwFyCz2DkAF_TpHjVpH{x z52Cm0!Vs4+Vk@0SMn{EFQ^L0Q4?KteLhmHw%MPXoR1uep^M(#Rw z8s2^x>~5w*%9xRf_>;SINmnfH=279pqApH>ua$xnW5tF}t;?of;^`4Glrv4SRz!_n zp<1mG&8y2T{utirLIR>N0f?Y_!$Wk)L8%5&OivTNwJW=(W7TVaO&m)L-c z^zo-kIlxJ0Jd`c3Y3+iMwdHi^HtYX(0UUB@g=Q_u6P>bK`M)`EymPO0)k0e0;M{wUeW=d+9c+>d#YvYN+xsUti8n6kfF+l-VPeTN4lstay!U#Sj_`T1#3gz7^ zPI>o^aT>Ir_ifswbGD}C`?U4B<-K{247A?z$?ZNO?)W@@At3g=o8jtt z23!|5+bsF+n~{*%vPlz|ZTttKfBmlDafI@9yGIUb7Og*|aP^${T=^UwX4=I*o_8%# zd0#aj8LxeXP>wSByzS+_Bnmw@iiFe)J^BpJZJ93A=(mARN#3^QjGo39^&B<~S6ov| zk=$M%HXOG0VmV&1<7z#DfAV)^1TH)_|OY9Th07y z?K}Y7d<@888+n}+!#N{983zq*IIr2YqHwBDt0G`CJbi9{=1>yY#$UF*oRT2Asd;%@ zUKV!)tclaSbt4F_&<$%LdS5X$rmpf~Jf_+s)_HaZPy2}PA%Q340r7bG7|TuCm9hkK zjKz_9(FXLN-FhXq=r)>X7-ZP@O@|(uV`qsXQl!ef<)2BXYDCH3Z0LzLY6Uko)=T}G zkj>^xkEy!?6^3XsXOl|qFsWcMIPJaLI=j{OF%EY#J{WSQqI^TWR0Xc$svC<)o|i3> z?^}tgc{WC&+p(Tu#U4cJbR(-IEkMILB&k~74mo!?qZvjhS3DVuA+1qu874-CXM|&y z*<^Xg0fD^2my!HR!^0WN5Z)=QOq+sAFGGYnU8agX;b4Y4pqXQu`99}5gER8~kEXYZ zimPk7wn?zyf#5E|-Q9u)m*DPB<1PvAF2NmwyIX)@jWq7=?hZ|VeLw&Io-r7(;h=Y| zs%y@gOIcWz%IEHA@?`2*>Q*Q3`gw#q*3aMUUzJtQDUXk)2-&{cZ+5f47xY_$rRNGS zv+o1W#ytHAoUPfJkaHzzHV#*&vC*%cBo_fw3)!DBz4;FQ_%kFIwHySb>-xMN7L#(td*zm?4b)X%d*hR81JA7^J%Q6|BguDt;~!m9sv-%#nRf(^ZA zzSIWiJy+^~p|bv3#=}9wFQUNUOUl9T9Ook2V18k|7lsj?p~H5&gV#N!0Zz_)yHn)d zWSXBT4+ly@?m%Ru_6co>%HZ;JbP@2v{$|hZWarh3u@2-96-N9sjOm^8FzG&5w(~*y z>Qc8?Fm)g}J@8?*_nyd+7v5%cXAtnd%nMC>BwP1*bx>zhq?jhyWqp5Bo^g~boK+H> zHQ>MaZMHh;z^*jbx?~>>e%!q>VR$u=3$Lero`HWHi zsY@%FNs>}=tVUW}uFoIW`(jtPP}NahPd?1TiOaRngo_>DfQt%5Ix%L#x9ca#pbq&r z7#MiAPmb5-73~J&H1w>uH!V^xiQ zyz+NZAb*s6*Yu?~^R3TWIv)($l=634&R_3mG<%XPm;$>tWrtqPl?f3z_4Ow)-yaLN|v!nY;< zoo{E}7@WXCz{bWn6a5maAB&-XKp6ThBl)k1e=rXw5#^oly_Z1V_u8Tv4(AHuZcZZg9!4z(S4m}BSDTTJ@psG@8{BIwT?&$(E|fxnTww$}?tM|V7Je-VC+#|^xZFj171=#T*)^h#-f;f69e(H& zhF64w3R#By1h1hTLeGt$;qN8BXGi%|~6e zRk`bTp7=`Pi26S5aGfsjAfmv?53DX^&h89HxtS=-%HD0y{gayUNLg!Th^xV=M`2vwK z3m8h&R3%)vW!=x-^Fach-zxX(tyADXI1^T?{<6}PPZ;w`BrJ7jw-LCRk+mnvs6I2# z-R0Y3Vp)|ZyOKAA_pMKLjuqRs>Qmb=)UR9oPc^1bRTb}_A+&}$7h`fSU_5G`A>~iy z*ZKF6Jr?FL;Q*MzyX>;lpGE8;O3GFtrP4P>!MC>YbF}o4Lw#9< zpD&aEB7pf%CQ=QxZ&^Oci4C_NeW@Fu`72{N~IZ+f@bCWo*B&+M;&^;liCqkTpEumGuN`7_Uii@O?E> zr`azY0e;1fK^?l(&$hS{x8&pqWsh>=K%b_#49{}4M@P&ld9!1K%M#MKnl-I>h>k)kPKRPwxtWPu4~ zba1>DPolQW6L?d|Ft;o`(Tj*I` zs}`shiD$V}_in~?RHdI;fai};uX4j48Hspu9@Kuo*eC#>v=62Rs&_hatT0rkA|wrayikChHO{{;{-> zCuJSj1}4U3DQ*WXJ^t)|AgI!Hbe@eB?YM~=BlWp@?;Dgr7qmUh@#F_b=gAqggE-Oo zj`9b3u9oAg4LU8a5r{-imub)2W@iy{0*239a|1858d((yY(=;C@SrN^l?7hjrlD?^ z`Dl5q*E;-R02m6MZPA>bREICx7dd%WyplUFyTA?dY3uHX=fRbT#9l`f;pm$wq8I$M zBrZ#sM&KJ83=yZz>EWC@uj?Jx-uG0^`F^Qr1nxD5M`2$5hV5A1_ai|q!wpT}r@2OD z;<=L!yFjPCY0!qx>iK2&>m(ome0En}^b11lvNY{0>c0u?^}HtLwG1Fdc2hP8d1TIW z^}goZ_BNYfjE&2m!MyAC(XSWXuSqDN&f7RW)mu_uQSj72+=}&XN$a}HtoJ59)0bLG z75^!5kkHW<6@qlvMMq_?|MKAVY?#M}_o6=rpY?E#XXj;ZC&K;n`(up)8M979pEF3r zyR38-&RdchMq%n$*Y-;mjC4|?}p#47~k{1 z!yf2F?1IzzuyQ&n{P@;+(S5j76ff{HLdCP>G;lXv5KeqkO%yH1}-Df64&Al7-tLgYxO;rN$!pT~M%IIC#6 zW%O9Oj&}M$HJxWSRaPXtOU`}D(EjZG=ou6^+WojLI8}cZWPwOhY!upTKAHsGH@Za> zZ7aQx9*o8#b({L-eJn%#J`U}ooL8-gwGX1B;mvZ^pUgMSdr6kY@HQYAh&!O)4~&Zt#}v1L0^cs`3}ja#c~5bD*bXv zFg=f&?saj59eW2qlG6kLEhKflgigzinG{4m~M*~4`4 z`fEuhYm@1S(#Y^vmO)sAcp3M%Fna{;-E1uY-V+TU}@7jXuB4qY(&z4WBgkU_No$Sgr}jPX8TIk#nLx znfiYH!HVoB0O$08PiXs=>0J_o{xm&<=8WQ3Km-mwvMMekxd@*b4JgIrvn&yAKGugI z@`3;-aU$xZUD=eWh(3DRQ@o1*V*#crB99awV-)F$qnUs|EUl3x>gg%rStU|N_AcHA zrQ@C}lg7T{t?CaYj4fSksnJM$z2=`ZqP?j^{$c3)n>C|?L#MsyU|-X2D7>34D0k1; zO6)kQHK_iFqqvntxcNUu>~p@cq;2ed0yWw}lZ0#yKMnY{$WS!K*>^ZhulbwTRkSFb zxNK!-_P7RS*;9{BkYA!=QuZoM8Pvb|)M=ldw^x#s$Kbt&l3vYaM*d*FG|{MfT- z23FA}U%i_o2g0ctm;z83T)7OzA3IHj%8vTrF9Tn*q=nE1K(a zXy7T@W2wlqhtjvbus?3Xh26f1yL6KZ=&z!CdghE1AX67j9$V6?`R3cPR_P+S%(%h`sUEM8^Dj=uEeI1&j_=@oq!$ zt;O_u8`wfs0QBZ*5sG=$T`!AzWGlL6|u=;w*o;zIfr&Cpi z@UK~(0wbf}6;U`M%^&{YQ;&X-j*@7VUO?ddos87ba|6@35CWky=S+iJU#{Z#nnL!; zq8tv0bOg# zfjq|9PQ1SAPt(F3uDMm;Pu!~HRs&~g)lY|(lLHz@%P)5$?= zVbut2MAhj}1raiKO0+3ZX=FujNQ3EI=+mWn5HUr=Y`l6}iWQPd^2A`BcApe9EeK8_ z*cdGnRiuym)a3?6%22`nr#Crux|Ml}ceYpNepDY;7$%^A&WwhdWru5qprr64?JzfM zqWk5CMox)~gDR%Yc!cnj18m3r$*LVy@tT#$e~dw?XmS)w)KS?n3Mb+D{8(b2r>myP zU!?0`htMg?u>EI_5r?t<$rdwSRx6l$ew3}quOUoAOuAqjeERFb8>FTah0mW%@R=s1 zUj9_#t#3*kBm1is&qkHKbSOYvCg5EM$%sGIa?Ui)5? zH9U_=)8>tR+~bJ&@k`Fd8+YJp z48Eak83IGZ<*zt$NHGngZ18J31!wkA`d0Ir!ex}1TANM|F+3@h-#X3uZImPKXbCMU zt6PMRy;%iz<@2B)<}rj;eYbaS(MOa^{^MuX_nkwLfZ6*+;kzl=*Nea@Vb4wWHs66* z9@4WozxP=(k`ZW8eG}Bu_t(NRz1faldDr|ygW#B@6GC~kbf@!vOaV3hUd@>c+b~C~$gA^mP>C2v{;5tA-q3LYjnAU3TMZ{0A#Hn~qDf zpanQs31l3<_uDGRYk?ro4m8Cic$qCO@iOOhbt)cC>OHTaYS!3`uG=?GK)^;2pSn@@gxdB4-&+3x zJw&jO_1V^9dhd(kqW&!yDu=f#D^6-5cqM?AwIqPY_1ntg2_Im8w9jhE`2~ed&~d@w4`36{6HWQ$^y1r z&*t@jW=iDAblykEbH_t6U>`V*z%CmN78*=52!IO3&ryaaLdhqj__Ho-E;b@CLJLv0 zLJTeVy!bUX?1joK{a})ouoS4zyc_o%lhGFBmlcZ8XT4 zI2Pz<+qu&fYTLLLBToWsr}7XNl#YRTjbv91QLo}4xWXp5OqEmAnxV|$+o zRR_!)LV2R9N5+55{0F56g-8K>l9ED}l?Yof!;e!P^AID8Yc?x0)U*sUeCMObfE9L}~oA@Mbyqke5gN{qr3F5a>R zt>WKQoV5(JK=kiTe}g#kVuQ)gIk+lx7?&L-P(`gaT6o}(D=h8vir zvne|DEr*e1699So_So}E%B7vtdw{5gwg;thmtBC|^|R6tYIz=?yTolR6F~jGCKm!) zr`vM7QRn`_DdJ83(O@(#1_R&Hr{mkJNx@5x(`w$9|A=|F%f7~DyWe5id75L}ZO!WJ z=2OS}3mLkf@uTqLrChkV~@Wx^gH>d0bchK7;CMwN}Vc= z7)_%z^1be(&&QJh1CQpuf12S@3AQSscG~6v;xL|1ThpR=^n>Ur?lF;&=F+OR zWd3~+VlHF78eV@)Cqty_*tz9^m@ARujY-WE*J9QvfJ>Ah{lU9^!07R~J1ag3=6 zz4#u(6o+97O8mN0N>|xf0nr4aC$uB_GbR*+4sxgM)!8PnL0u9ma;shoQI@xa;`FEq zaoaQ_^uLYdNUCv=3^lR8T8Fp@DbVB5;x)Qd8P|~Uy^_$#aS$|V{&k5webLgjpejwc zbZu+qX#`693|a)1k+lyNjv%Qrk|Tcf!s4&_7=H4qhDuI9JcP|k{vu}zgXz&5w9L%e zfJlrU_f^#1JrXqX8$N|jgep@#g**ban5>iz`>%I(0L>E87R*a5F{bJ_nP(xXE^1AR zA-JgPUlKi8SZ0``QEWtAndaO-t{NtbzaF$TU^%8=XWeDRVeBx|BOeRYdZ81L?!s2q zrqY*zOHzx%`VcikL6WcMyT_E>ZuLBNf+3C0VAnqQhVzH* zHB4W6?-uWXkZx@9#o&J>FLBq_h5d29p#aFGyx6(-k+u1o9@0$5$NysiCodG2C8h|& zxt_S}X>z%{%+SoUphWIc0;7+Ya-`Nue~SAOY>~F58*<&W$Yu4WPqV<`+Wv;_mj>}0 z?z7UXmouocLS-CprOCg)VWsG>Ze}I*&jN{RcRMdjTUwHQRv|k*3IB?Y~cI2}_FzRG;~7Snom?ejwPypXL%@{U*F>&&l=AV-*8py##Gfb3`(8a!BT!^q^*> za5C>T@q0%jw6SF(hpua5z`=&^B@g_%=Oxq`B&nIN^tm{BEj7n;1kbUe$oX&8+$LKX_2-(MwEz?SUo{UGw2i{$Z* zoC?Yd!$SR*QcG1!)P-~|vM5(NW!2G`k)PVMh0tm3EdIX0jdU98P@3tSG}#AH8&4$Y zaQ?2-Yodv5p?Pl4^TN|*i-2#p?fYoq`n5*yGlKfnC&5}wY+vpYCTc-~s}&wRT+b1p=qHAXDkzo> z>eudOPZsZGjusQ$wgp6C(D^cDnx8V$sIZM7+(x4mD)>lA*CbB((e7i%$mTrD zU$p}@MTs&r!Wx_`9d5OR(^b=Yk=tV$^rgo&(UbSv*j$CxQ(H-z@%Dd$XpO+#&dR9Xr8M9jjeHF zIenR^=PePuaq}xhP6w8FWP*wzxp;KRi?QsC{83$QJH<6fIkrV+@+HX6S8kQJal%32 zD_&uS{?#C;6m9-nz5$UGT3zz$$5U<@T=TTnCzs&3tAg1m?8H@y|Cdu*O& z9cIsq_>P#!bRSXI3LTz7SAE(c-cQ@%h=RzyFIj=IIGGvKIWh3$fcASHfZyzD?aM`L z&vkQP`{TQ>X0`V7g6CUdri6E2{(n})8AYB!#w`1eU>p#tdtQCC0P=yt1y zCOS9gG^n?EEq^?V6EE_zto-)411*gf>FXE*dlBkJ&tcR=j&8?Nhxhcn{_D|i9mnUb zXb|0%0St6V$jXpBuU?Oz_luj#z=LunRCh{bO>s2p@mo5F9P(DIze9zPYQt{ZDdwKF z<>tE|B?&GPx_IaeU-HNGb5m&Pev@~1hH~Tb-WYgp?pJi*YwFQYdI&X_Z0kAfD@O`H zO$AqKiRMzS9GB%iL%Jct$FtD=V#oM(&(8N3BE%9hGf#R1-aPLE9S8GG$4-Zw74=+C ztowZE4{5&k%|NpTg>xwQIzQ*#_HN?a^K6;K?LxKkIV2qLDC+Reo@Ouaz9FJ*nEM|(qYxJ>r$9_Nmy==s9h*~RVo{90tgCMj1I@?H zE0`qVx3CA7_#S*0t*<$-zHmZ2tyAkRa&?VU7&@V-TTm!<7xoFzzqr$?d^4X10Cn+- z%6ItD>TB+cHC}TYvqW+^a}hmca_@y2hU)BVu@1MMpI;(pMGLFEul@ENdc+1ynKGwb zW3=|#Z6vWz9*8fcEW=KPTz0ZKNqSAUh9A+JMpC9O) z7KkidT9IS7PA^X`{NA%=oEXYMP+APzm_%!)ECKz#>^&(A2ek7rSR+a&(Vu(3Ey0Ut zWE!9+hVMqMlyUIe%6C#;F|BQ;h$LveUHQfbzBZK3oSSz$co!a++UCE_qHjUvLigz% zYAAgkJkh9J$?L;{g5ly+nVy@~^o!2TU#}OQQ^6Q(qD%TVwHF<)8;@)Lhwz7au9w1| zP&8sXoa0f53?=Upg6Ca!FU`Wgre|0>FWQa!f?Ov>9P&N(&ElDLySz@T-3J?Oh?#Y| z;FyNLw}Bt@Jl4Dysfms})6Uw5XD{1!H4Cq44m~%#U!gNg-R-Q0vev`KA%V((a3*0c zI(H1?7}RDgc-}M?Mg-{c*$4hN1s#gqZw5m@3uc$O`xo8u0rCBoSq}C0JgXbt!&j&O zE;~*ipiaZH_Ct~5*)m~=R&g{yr~lLNMDZ%*5$Y=@wI_vIg_UwbuM9lbp|C|+xfjqd zS}JJYJFD(}UufeiIWHh2MMF|Y#|xsK%n%A&HTFidp!-jDhkme^qR_U;$})_Z_bwnd9P8+tYG&*oskAgp3&Wi;u6(x3+9x#POX&LoS--X3i@xa|unv@#- zWgve|7q_j^Mm%GM-6b~JczN8vQvYQ448QvLOcuzWm+G#~xgc4kQ-+Bfqe&G$INU;x zM0>bUUsPUpUI>|+P-3I1SgI*^B*a@-GFu+g?6qd4>;sRg(SHn?Uicq~!7j%4=GYS> zG%ZxX^>zD7Hu33mV9@T~bo4x#%EF&XP3bc@V#Q7Nv8!4w79L24mcr5rz^3$RC~aXu5`xw<%B z$8Ya2D~W3A-yvQ}2ct@~1XX35pW}Ep%b$&ZYnunK+0Mjkk1P${L`V=LR$&n%=FO?m zIe!GhZTu!G{aYz9qNOWTHxz5h!%^}XMzvqFc>0o`qSb5TA0@+;i$sxSbXuR@TMg`! zLKMakbD#6o=*}90srx!e_61FIj&2))F1=m4Smoe*<9-SOkBYafb#VDmvLM1`{K@YT z`E9Da^Nu$ub1DQJs<3IR+DImiVd!x-U?`cd56l-aHdV5MiXS~ygVI62p~d0AC#C&Aq4Txs$okNxe{ zZY?Qc0w=e7`!4X1#%vsF`Y;c2_{KI!vz?k*<%!-bd=nav2`ZHU*((*?hgg(X(&Ahx!$sfsAhcP?+Ml zZn=7n;e9X--7c6HQ5L;QIn0y8>9{{2YFV>A-BtG5HcqyT@6L06i+r&_^<3v)`{uQ9 zU1r}t&jZChk6OJK%ytVW+mVQ5ZI4is72jQ55ke;k3LdkEZST5Y<4C2^ZDZ#}&*FpN z#oHTRuW76BgU}_BQ4#y`5K&F{^8u0Jdj+qMS#NO5hDWDao_l}KO%A)tz~Mf3c217} z@tEN)C%|V|Nf7PP{lRb_tOb||Zaj#W_g}XSzB;>RsCv6%~dD#xicWv1;DHDG1d8ssnq@`943HIH7kqvrS!N-BA_+ic=RI`={q(~nHo>{2z z%NRrX*6iO-{c2@_?RNTNuk3#eqK{9z>%oggff66FU_1UJ#Sn1ib=Y>8-0gAwFmdRj zvgmGANSKxoPr)i`i&!Cc#bfcC$^KOdYffCNL-)Ll7kx=sm4Tq|wiMp3%y~ib0LEhPqkCWI01-?3Uc((XgFMgD)?aYA9{ua8 z=(GFA^O$*@xr-=fi4NDkzP?k+Fs|^dui3$`sHYD($%Rg=Uuv$@GIjdy*nfKepur*~ zswzcEsFj)&{#pPhsQl?u7N(FZL7$zVSlq2}l?tU!t<3Tc)?fDjAPaZJI{$KxlR?qv z@*Sw#vw+lfNteLWB1vHgmReJL4EJDs`gE=V8_2oT!2P}2<}gh%^qI6h!>}(iWC67u z9&KCToeo9qX9g#fxePfrr7_F$ZI0OwOV=!;TZyN5CoYXG#mTkE=X}Ko-d#Kot-n0)>_r%<5)Uf#S7avaAJtOC}AD@)>8&LE@3-=HU#D z{Hp&LD2E&FXhsL^v5F_eC6|tA&f#?ovGh~6sZ;4K4*x8T)dzXt$tC;WL5-&z%RCI~ z`ejI(H(n|2O$c~$P-(lAK=7wT92^X71k>6HFFvIP$U)c{Hb>DjW zQT|A$=zdL;%traE^m|qrVT}mfC;sIN!9QBP?{6}c5_cK;ai5Zzf*PzD(s2HohzH(C zH%cU!V(KV;j-DgZ3D3{E{|kia6vl}}Qc|jMFjd_tAdUWdK!+Y9fMf<=QZQOhgq23~ zcb;DJlhv?)6*+^+kDy7GK}s7oiM|n4(T*bEufOcU>9bt}`&B}Lm^kJ@myCHbh4z#B zZdU>qxx^4U9e$sxM1#2*eyVhS&%;ivGE>rVSJT5Xu7i6_0r`f7I-^c~fJ)D4235Ek zNdAr18il~zw7tIP3D!sUhHeX9nM(I}ivqJVu`S2H9XbUa-UYT&u{2G%T(k)q#o6cq z(YNNel}bm!%V47uv?Gmr|4u|!nJZ}TaN6C!3PxR6a-kNKV)eb6BK)81bG8rXklLt_^n91$6YsWS{}ixy>Ig*t(!q)%tr51 z!y-4etX|t=Mvq~6Ms-fp#%0^LdCy0&i$>~fKC}}xIpA^H_f2_a|MM}!=NBm`+js|0DGDuoOmub zO-$gniWQ|aa(TaL7PJBo6()Cwy5=XDelJ4}@-54jS#Hi-0IS^lRIb;K)0)nQ4PVwD zzCaYihPT9hE~Vi`Itx6;TPU+>^?P=4TeK2BZL=DVZah@cvCHB{%_L^B)abnrDRj1+u^53sq)Y}jf^My-Y1~G zi<2pd9fb338KGOKF1Ti%tT^NFDCz`(GO5x!mU5194YxFTo98bFty1x^Fx3M@@}z$@ zr1+x0C}fZRjEQtZkV^T#JhD+lnaX@ov065y2HUU0zQM9^z&e$asA8jx{INW04i44Yy*Ja+2GPt{#UU$EvJE&eO-2HzQbhOIv zI12xOT#DrU3!#}mvfQHEIp49>6e}b7=T5r)J|WRkzBOsD&i+q!K@9@Orp8ej1u2(q zXmf`$$-~s)(%`S{A}98)aw!z8Nvp4|FGy~LI++F;i+{5}Y2~@oQfX(n?Gt{1R?pH| zhZV;0738Jw+nB-8<}Sc6;@SuW$8y@gj_On0a1$-aL7yqK0g)N(K1F9vD!RcE08|`s z$AzWJ2Z>?YH-fGz>+min31HQANeo7%@3S8T_c*z-Qzy(+A7%a59er{is)7JDFdI`5 zG4j+u$*?6I)sK1CPn$0aQCK^%;~4>0-zV?YaS6l%DLAy3L5_6ZD18EAP8??qO&cR0 zJWb8tGSwLB3xnnz-q>Ak6xI+a2{NN9kro3-M+-hPMsUO+hbh*NjMdk{z$f1-I4J@R zl97m$yP0;i%mqaYnF29^g`^ghd5Y~-i_2$`YTsUI(&uk8G3xEj@tVz*dDBpyxj>#d z?aDRbx}X*Nh%yh#tQLh{57-*r#lMJwes|lo~2hA+Iph~+iBJ}SCZ+ZoETzkw1Nk->DfG?sPq!3u5!j=QZY z&XCU(42)h9I(!l|KZQ@wmFn^~N1%ZhmG4jS0h{v>ShBbE_`o-H`0Tl>5B?X8WTH>` z!B>gHq`qevGNi4x`*KyFNx^fs3*HHxn*Xx6!byCNF5eyyXHrq1R0Iv+$2n#O_0a~t z&DMJHi`=Y0%?3Nsto8m$_wj2&f3}H?ARVG_!JATO|7{z+H1<9*jtlcc#zY~rT0J)w z$B7W|3-88Di>G10GQ6IBk5tD~=3@6X^bv!(qOW6RmyPkyyS_4HkJ*BkZLUzywE$$T zcK1G-rQ@l&7D5n|Aq&(&OJ(jEChY`Y@>|hT#|zzzxGH;XSfm>+y}wLJVEhNJ6!Z6u zAoq~l$yUfWU+`INz`4AUz&Y9bJ76}rEYEqNbMsbYm*3_=qn=~Dvje=qWsL=3x(cau z0g(0WzE!B)R_!*EF*vk>RdxnhjkQ|yM|skBS+P=Iv@?*I^ZqqR zN#*b`>$MCt50@(=#?EUMeUY*{rPM5qd+&P@+c>WJ9^Me(aI7i5L+iXS>|fZ2|4>N( zA1yv9Wy=2n#s8`4y2jCu3JINz*VQ^Qbi;4nbMeoK$hV=R_lrJ7ROzV~tP{alG)Uft zr*a|paep!z2PEnkY$!&6W)BLnhqBTSq6$5^+JVh$=l}@&l+k^*<&~L@4Se9-T3P%d z_8qO^LmtD5ek+yTH&C`6{jTqOM8ko53Qk)bv?FPL;zYP~BI%T6)06~ST&hyZtbKhh z!2lsl2pzUPSC~BB>zCS#sLeJ8cn=ds=S5ZTzAXB8U(xn{{X91Gx6V;PGfwYQM=E>O z#Z}*SjYKEL^t(>ow&CE}%5dP_W?;+ByI{u7J0ko#u1D9^I;{o2WIKTd`RZ#{q&1^x zz-7)D6FEF) z+qsQ`JNY(m^j^LSeVEpyh}FI^Z|F+mhAB$Ne#>f2+l`>tDs-3au;;+PZUID}@P%0r z^0DmsY9O=MX^lE}3;ZH^akls5OKD>+W!WTf;;j2bz}h$>1$i5W6O zV6_F7A#>mRg?JI)N$$1;j#0{)L;6SYBhO?Gh!zfKw7nVmBOZOdzqqWnk6y6CL zqVEAGhO1G2gZ(l*g}aGfV6{x0&?Md0Y3#L0XjW!n1zA`UVuk({OZu^%5%OqJ!dUk5 zjW%OgsV<0JsOLUEJ2|h;SAxM=T4!xxL|hndqo}@1n_^eB8Y2Yv9^n{7*XU$QtG+X& zvTy;fmkgX-P-CpC9noM6t{zKk)@0F1>tL1_wg*y=4P{0s05iT##8I?bt`?ZMHMn}j zbXkh$RO3LmcbE^7R3*%-4lL>kaqRKH;S~~HvmS6T8bhJ}}f#u_L$ zR8E{m#=$#(5^<26zWt2+;=NTyvuM8`f_Xt<908X974(@USKdxdT|-4F52(0~_)d(t zhD8iw;i7OHc&+^Rhw2^iJ zYl2~**9_37jKq|af8pQQPEVmkU>Cppb93U?rQ+yf^~F(UkB>}`4oK2oOQSBT#D3v! z0J1pgvNC1iu5HHS!zrpi*NE8&X^3R7rVm$>YoSRw-B5e%$k=lta1IbGqg!GClyD)1 z8AMqIe|!8v8o0N8`8F+h*>hyCZTi!xJl5)qA$_QU=CB2imTkiZ|6JGGsg=l8Rxj5o z=~)D%by&Tvn*IB_!K^7S8{P}(LC^P+>0wow$vHlcUf4T=_J!g*CbJ<$ssu}N>hf4| z14)Gxw$#tA(ZZ_;`A=+9(8!0k*1*^JnhAMa*7x+lm+;NEd!u#7RfhtBV27|Gk%wuB z(0!TV&PCu}qRjhAEhLPxC1Rjwkny7OSXpSBANqDlzPLj@XGV_i8|nFIFF-Aj^J?s+ z|D_;?;Mvqubd&R$Zf?9c=IODcSpC zpkp8c+Vg8KBx}31pm^lF+dKT23J3&w%S!f}rcMXyTR)xO1?Q|hylCn>uScLg@BdYhJ`?hP{wDeh4z424^|f1A^@f3G8>8|k%QVKUrGeq+T) z&GG(7W8V=9!4ep?{P@8_93t&eiLgSGK>ybq9Za!I;!pli3`FXb>?+|%%-YfEY)quM z%dZZu=N5-^#jVxN_&8p>X1}K0?x<6tlg6T3=KOywV5&4H4UKwC+71hcMi8}=a40TY zk2BWIoeLmtrpU8UycSfBZY|E^k`9k_;4ZC{%emqaQ9yr7DR)F{FB8fgcO(U~y56Mv za9-8@O$2M+YUEkDA)PF;z5D;e6Uk8#L0k>o99w!ZtyoJcaa}FCc=Gkv4ErR^D5@$+ z7HI4TnovLiG}8~I7;_lsBA!k+&ecv6chtr$LAAKSv%{L2gQmoLeB`sRjVF^8Sl>el z<~jFc(EWNsJEi>d1KlhDZFDjuVUUY}C`IkBs$>TX*}PZN=YJAa0bBMDsHn8b-bEkL z^F_VM8HqhQjXd%O#$!fmhAozj1Vy~~oy9A>bK^IfI6Kknp`!WCj zFhtbkdt8FTY&22~^-m-W<}SqT$Mr$P)6plOy}ho!(T*2Ti9fJSkNkz|(Um}Moj#M3 z-^2#GY$vbawrtDI@%zgj z3hQg4ljE>nF{{=_iainl{cTfzp-$G^yHr7s50DBRgOow3d>C~&FI55{(J9HHcY!( z_uM%Ld`ie}rJ@B57w{2Ae{KvdEg$^sN@~#1p2T8;r7M`7)P{6eHNQ7^=SY8=n3xnV$P&dbeqAh;~g zf0pQR!ww|g?78ARyx4OvZ}ir0#k1x_)ph8oLkjJGiaXtnQD8IxcSC-!I<}{(Vte^4g8%S+gFHzz})b z6n!h;?Rsza0%<{;5ZGXJ%qFg>7W)kT2I!K*D$VmwG>g*m%d`v`_(Ee-p3DKUb=_yx zfp^^{+Kp{o*$qM7W;A(zR1$%Xq&JXMg3*Qc^~l(2BnJm(Qpt-5PDu~a3ueaOwYEAoSXnfxWftw`IvgBjHgUjq?g4AN ziXv|mLOTSziM1es;AVGx8}V`sdZ##!BD~1+ZZl0g9JF52EbNEIm-IRoVvc0T5~8@fzl;{T)RESTDiwziFXad(H{?(Xh| z26syFP+W_<1}N_C?kyA#?oKK0?iBjvob$cEATycS`&s*zYsvFY5pj`$%BXk&qtWe| z8T_{_)M&soWihV07wy3qnlu7)P1=HY^QGOMn%qg#hYmhVj43y^qlvw$K)5i}BY6vH zn0EB-B|?K!Z+;%)0Z2hUxj)-4@SaxA7=W|@+VT?DDh|`@moMDw!GKYj>86%V^Uw_G zIh(QVTUEDTqRWB@<2LDpq!JKEke@>0kSq~*_I{nk`Lo4P7TF}vYXU6$PPqC>8>8j% zsS8l@689P1L*h<_n+#xWwCC!Z?sZ&sb$)e#V|KolirwuoK>q?5J)tFTJD5~D( zX`>VRE*8Wl!Kv8-TFiYS_V-iTi7(lGO#WyF$`g#Cezbcjs4VklZuP^N93C{!wiIYF z0;m&x@M$ieb2Py?nsun_Ip7R@R{VEg&U(=!|HCJb@18M}9=|LpONW0yXIpo%?Ipp8 z@~Qo2!1iO1)%oL(^&Ca}D((&c%rp_fkLb&}QvF*s$AWQL2gR}nK6(Y7p4I1EhgSb=KI^IfcTM&74z+e81pV)VB>ocyMX0JP zP}ZOw)l7NLN+8qwqgtK|tUiRM8LrP+f8-W7&h<8uv~T#1Q~evu@18dOTSWAC3E60mgO$oGYBLroK`##xU^m+kAQ{rvtuEO-+*atC^ZOp`ROWxua# ztO|B5yhe6)BGQCnt)wM8*)Gq)f5Why;Cz*phOG{~5wB!}5F<@C)eB*HFP9md{E#(_$apY6YkJ$+A0T`v9 zO#8ZU1^Fbt`}kUTDm46%u9Ul0I0uo}a7$2^(~mSoxte?$uX1(y6!B-Im^K16fiWqW zk+>un#I>PDY&YbLBp zH#uA8cGj@q&k+$#Ie5ie>Y=c)Yxm>pdr7K^!51q+5{xjVil`W#uo9)^(PX@BbJ9>O z@TUK%lcdK9>(ATG>S6mot`$)5+1mi;KS({C!YqLZ?kcogB9|!npzFcnTafR50 zmiiH3y)6d??Zxr_%=z7D-VgV@Zl>a}R1TnLq`x6$#AB|rJ9NZb8@rP9? z0$+!a{0gdPg{knZXxZDLr@QaL*>&e5p+Y=M`Eff_^1%XOf9fKQFV$C$Xu9lTc`gvrXF?f05p!t8`RHvUU_l%XY{%ZKM`P*Wa#c zrA~8HT=g&e+}d;=)lBf=Q7SW1Pu@ke;zzT=m?!=m#>V>p-rK&~uqK~7?5#Gh1p1k- z3rQ27Gbkvv^6D?-^Ki5NV5_*@C*3Y8Y+G+3MsFO5;u$cgW?@h!tsB?Bpx)tuWTp^J z$Jg{SKYR-WH668p9)cr<>biN|mvEb<)q_jVPkNGt-6H1^48lm;ebNV+J_uZWWUO^u zppq(=Z-dqEv*8@P#?;g5D>D5lEm4pO4sB1`t;tc{*kP3^F`Uz64h;ku$K!(yq)`c~ zo&Sb?*_b36Bzsjr<#3g8#-Y>wCi>=Ch9}jW?BvfSHiSH20n_;@Orkidcj&5rTs<>{(?R9vyQD^KYPzu@XkPYq5E&+20^B%D%}SBTSy zwFD@9lcJ7_qp``V;B9dCHynSLbbLZzG{wdm&x*2bWNE-e!C&bqujR zo^73ewoeRK@D@<^t;Yy^^<~HXkNQA~uuR|>(d5tbr#MtvOo(TJ%(}H*`?H;Qz{*u7 zO@7*s{P;KSzt#n=-t=WS`+-?bhJvgn0_%?36NC?XA?H88a+~&73s~z_c&iowYN<{v z3IY8UM0}MX{B|7eyU;Yu4 zxLQBVxxI&Brja_(xB=P)?0W?rwu5p*38-EUgEs8) zdqqzgJkQrO15bFNh25Yw+sT?Kv>2q>n)LBmj%EitCL$|}&%M82Lv4tRweI3?IXhf0Yp6K7j^^$zq*8B= zQdA?M6r-W4bpk1aJ*QmD5q1a^D_PDVBe4C%pUntNKmqUhsShN z=-`w(dZ=0JeaDY{NhSLO74r3X5$6vByi1+gZ?rl)gWAeBtJ^M`{1$h{n!rFGxN%!I z565xb3_CX~UlZY9gJ~gdHLJxE;+CR_L^)pw&nmEpe+$WsdG03{`T8y@I#_f(#U<>7i#5XHHC>q=!@|a7obC$9n7N0v{gLuub9+ zezO)v)HWNvrLcs@Vr;1qyY3+4UMK9O9WVgz_N}No0emI0}k|l<&M) z#omF#^|OlF(f_9fu-i;LakU*qL&VmW2^Gfpe$Ak)D~1h zwf!o0LYH0faoS;3Dc?Pz`Y9q2M&OZqT})}8nNK;;|K|Fl6&#XtA7^GoBbiOcpPHqb zi-^;iX|g)6kgrwPqRHiml6pZB_R!w(#>JYCahVmL(XrVZ1x=Da*|3~jB-z@B*)8*6 zVMSe?nLpaFe|SEVa;}>ez?y_yAdl-_Irt519~|@_To-QmZy39r%!u)?`zNZ_80qyn z3zlXIk4S1U$|hNvm8ov{v764V6Z7RtX&#TsqlW8B2_I}5mtCCZXt6bvJFbVbgHHDAUs}!n)sFwXKmWU`NELABaQOEU-Rw^H=d;nz zws_-2cI30X!b z-O;@#YUe`ZdK^84J$*xSBmt?ELNs%*Jk&sv^0W|H^L0Z zf=x99C+ZD4s|!O%$i-iN$u7_c%|HO~xaL)1tu6QomE2w5=^f!*EoLy8{?fgux23&d z&=ED5-AHBLCv6+eQb`3#z4f1BZO6mj$&R>p6*U!6NY>NPaykrSoy1$#l3T!~z>;iC zje}>jm8|fCjqZF@UWSNa6yy@NCx*WI%^IgCkAfg6~MW->fABIbC$m9b4R3ap>_p+X;w+G%!8u}nr zf-TtOqnMJ4#~ns~P0pOr)P!BuU=)=w@yw4V#! zzFzUp8(Rlhue^}>xO&@`1{gahp|!oy4v3YkIh=(zGXLYxSd++#&yRRm*IoO!mW$)u z_G!(pG`lE=7+rIh-Q(*5f!!02`M&%&0fGK(*WDaF%ykRERdY?g4c^%NgjZ61^0y3q4Fbs)qA{;_Q zG5HY9@jfWFZPhv3#BV@WIrQ?gk=pF`j#hysHtP;uBC3iVLv3 ztbm?9M+UV}Q*zl#-127BcxF|y_jLXo?#)TjB*Cm&lUw3W1whg0``^PdOdskmb^_Ch zootOmjpdC!@Oq;%=q{2BMQ_Td783y-MxR>2Uf-+cj4~gUx(l+DT7? zm_%>**_LCP+dx9t+k^3Yd)I=h?XcWtbNO?%UF6~S`^b_)2U$ch$wAd~1x1wWcI+Tc zqDY&;Q^z_i2s-b8iww!tpHc<*(!=%9=_27UJ*FpvvAPpJ^CX}`@mYAwW8+yOUl2#w zKC3}ie9{AlH?8S*33gyS)=-Q5nk2|4*SP|}+d@Ux(2H+F3wZBMchH#>dsN$XAtvF1 zx<9K|XKUkze*OEUAo*=6SsslbQSYMQi(=e*ac}m%c7|>7Y-XiquJ7w;7{`9?{JfuN z^~N_ICIOk*CI&g3o1&?}?*>?7hMV5um%+*b1slOKbX{n!g!wCO=2KdZrk5PmMr{in zrs?O4AwN8B-K%~qA^50e8>s6n&CfsWzu;#li_g>-6<3=nNtc=Y#}zB%!wG+>IwA+B z?*dU4xeS(dQrocGAE%E$ znNd#5D(DF>|JLvG!<-CfqylP+yxkr=y9acHw4zZ^+k8E92aA`8ELIN|J^Y@#$S9AN z`IXb<&F8@vxHZnRza;9Lt8*b$*u~8=cimlo2pW@AG|OOpd$b?!Hl34P&hi>>rPSrA z^5e*#l$=~)X~eT3gLRm>;5}*!cb4Q6Ygx{|45Bi%e^*<+da3c39Xq?K_Z^ZPSPWcx zf2%9;^&T;N4mCfgf>o7d3zDpv1&Ryn7V(0@sU*10K4?i>9$M_rog(*Ve%t!m88K6Z_D(ok%O zwbRk(l?zgsHYth;ctoBUNk9G0!voAz=VX|Y?~IR>_G(jp7@%Xy6=xLQYidc*@)X;T zqeo^<1%1YqWu9d}PN@-V5o=giw?E|7V1%UAdUR&?UYcj?b=4R;kYKt}Od1$xg5sLK0)_R49C~}i<(%#Xs_+TF971POx$=Svg zo{$Y|<`{&f7z8uvPQAWLja`5pb5~7zTLyM4Pm^GIj}bXWa`f<8c*0AfZeT|kSTopI zO%>&4z}C8yHbA?{Pj&E2#kI1xgeE(o<05Mw&KtI^wshIwQuZkZz^PYWMmlS1RiYNKEauSL**RD z0I*l?FmDmMWQ^gj{-n)D)%n+(Ml)3Im-?W=-SH#Y5+HNt*g6jOUo3X|#V*caWnq5bn)10Uig+x4UWrN%2$O z!l2=L5Oy+f=l3E(!|*TTFo>j5b+9;op=>Ycypu2zsn*wj)e=g>j^YjV<8a4#b^W<; zqS-VlQ?3Z6LPsZB8T7T{$c!cMq;}*^! z;xYEu{3COCfU3wEfE(;OVr=8-Te<_DitI$aEZNGGj}APV{=FqX^-{Geh1--2v+iLQ z_5ZQ(xB60K6K{+EAL19kjgjsnkFvYgn`#r2VsBh~C% z2{XgDi5TS=6@Z-eM^ni=1On00d?~}OvAGfbT1;-F6uF6}nw1v!WnWwph0HWyBnfY@ z<=8a>7Yj@6=&dwhUT?4F_jmkK<@Xv=Yz+*Gayz4J7{a^uY9r`lli`o z>-N9YqH>PTuvuKBvS{!Z@a;E@q=?=e@l~2~iV*#T80vbPKFIK*mJ)@J#0QU>i`%R+ z6lkm~U}xN*)})pGns$%=C%Js}FoT}O3QTK`H z)X{rQcdSd!`ot`+u`(uyUME6{xhb#WiXMLfNqW=6pJ3B==E}`!%y-?0?yq=oZM0c* zECYBf^k$|!e=FWFeVl*dnR>F=`qYw(lTb1PMfYHTbo_X;*R0;g3e2Lkr*fID$C$1d z9AKjl$8?!(vQxY$Yx$6_kiEruStf=W#rmHs6#@S)pc)6n3|S2nZ44f0GF}UJcK;}y z{&e(v2))ackb2p3RIak<%vvRj3wo4|jZM#M1DEij_yFft@JS1%yfOZ*pil&DEM=I_dl_}{*s_OfjG1c$UBF4$u z_`JXIYr#A=KJ3G4NN*C7DU#VKg`yJ_8cW8-O~*3QJjY>Zif;l$pEVR)fYOX$9n0(q zAyeA9#~-aVGEEBzwmxTIwqk-SUOQqk9{!&0BrOt>-`G>z>~R~^rBb8C@3@$3(zdg| z*JP|UY#MDJU{=khYLaMSIXt3A&yAjkeAEYN!hN*nHXt9>Pg^zqPYV#0pWK@+!=@!i z^VVPhA6UwXMCFnPq@)Cb6nM1eKG4X2E}KK4H_3^5L3c-s!lkXY`^BzbW-feMy+Ns| z@P^H;2BwDC$yUkfanQ|wm8_OhmMcx*tG9y#MfK(dd%LQk2w zpCZxq$CK1%UlNM{;rFWcaIkG!GR6OrzOSkhDr*tSdFM_IbgdLm zD+gUl%2~gp7*2O%7`Vxp8HK0_d1U!GCn)Mpr!z&(c|#g*m7sN;{92q&u#jI z;Mb+K0YS!kBhKAtRrh9Zwm*Yla9hwHKNP%npQwu4WS zt~w*oY)R~UZ^plgF+@gRp;JA_ugdkeIc`0l&xrRelM_|lFP@%B+ycgVI&A}u+jJs* z73(EWh--7{+P4@On4T(9DSnTwf1KqG^wGf+~r6U^+M+N7~Xg=@e?yN)T&n%9?9Qxjoz3TKk2iC+r_1ogdyd8^D2;B^P9CH@ zZw-*=9#r3{;;FDaS6DGtM-Q+Q%+F zU&9c3sh$?Io5o2xO*B%FxS`e!Q7_gt#cFpcp?%qRs=lU@~SaXLZmYcEhRP$cH(qY#!I?O`XvP zFKfaCJWXR>+Dmtc%qcZ-|=Bf zOTepQao*iw{ZVLkx>^Y~h3_yO%)Tf<4Q0k(B;Ss(hT)aP5UgihlFBoK+B1C7R;PJX_mKRy%PN*<$a(pFii zTgKjTBsbqX^p&~F;(5eFrV7=X!v0i~q0Gl#2X)aQQx8%%Q<TsftjO zRw5Gt@}CA}I59>$qCQFKae_b40x8Ah%v% zulUWcrCMUGE@{=R#+kJ}N>tZ)*0b&ZzqmE=&wiPzGHaGV&ELVgf@;T4goC{j*PSLr zNx$VVIYtm4FQO(BE#y?jZXK_@O>wN!Ihr^In^@ShHe_2l6}+r&jO3~xyJ?g*3y#x`cY9;W6I2lo z%ci1EZjJMH5A+l7pA3Bbhni*+s#2_)cGHOB?Ju-)>wdJoS=jEvR)j+to;HPaAr^l` zx-aV{o|a9*uT^g@J_34OW*P?y#qiUTbe3yAq%rxOjL{nS0Zzw+g5wD0Fds$A=OBG* zp1R{|;W7tN4L|$3$N=TiLZ*%gLRE9d&JtM^53}3(2P&^aOy|;Rz-JPFp>=;jG)n%$ zhxGoTgJK{k5#xz%q~6~C2Tv)Wa29u*QMb24`&X^rs(N8UoohEh@_Mj1u&gD|KG%>Q zPMFEP-FVH(na*4~!V7K9#HNQ*%lXSJLu#5;DGBMooE1IYGUOxWb}nWb$rfw>qq?CQ zReh2V5mNe1Q`u2AKXrTO>NttcHzYrf{ncK#JJD&?yO|Io=b?lc)oA zPnL?;R0`DOYfuAair?scx*tEVV9aO)aUG_PzQ7k^MNRNipL$rzsc_*4pQE?QOqAQ2RL`H88GB8EGlH z*-q|hM<4e0t!lr;pts3pA7B+D@*`Ncp)-am4WWa$X6eY%irM)8dxu&jBRPaNqF%S% z5by02^HW}z?-4DvORx5Ft6-mw76GU%o0|dsV-h3Wa@q-YaE^6qhw}IUhLXsN>7xEH zLVRH*OrA}JrUX1Spj;Bs=pM?9%RYFQJS**%SwjKe`nMn_)55~>{@H;mza5XYllNgL z+M!aB1`Ks=B(-1#AS%$C3Mj_Z`)XDRn)}~C$!+bkz_|st@lbF)va^a0j=iOuxwLU% zk}nK)8ZD+6l7Yo0-EX26!3}8!ti(Pz#$<~HPw#0(K*XALzEo4|sC-u3Bc`fwwUw`i z-&VDS<<@sif%QS6m3&O@fc0J!6iKiQ`9#yiI@FuU(2PpNs&;gqKz3#`t0`w1;e#;B zNIzQVxuY#fFjbk9TsdK61sc9B9Cfx#lEk z8H!*zoXUx6b|FfRQ?tp8;n|xPYxlHS_i*oRW(4K(W>$k*Gn(=+z((IXeFfwdzHTlZ zp)Sdy<)!+pHRL>mp;@qD9{*0_X#7e*$&Bi&iYxn4@8;v@E|wXRuTo^MQM+OZzc7Ti z`;$!uL%J|tMtvjNkI@ZOi$h1Lru;8cGg`?@_bx%`t1L^liIYrim!?mar<$gvZ9nt3@%1_8ej}u_0G&4U!RpZWs^rO4OC7A<6?3iZZ0e!mtO9UWvrmvm z@|ev-4gC(3ZyA)(<~>xHGsNWfl2B#f@G*u`Tk{l6;gGy?c9(s9@kr(8WF-IkaN?pt zz{z|wcW;%bDP0qeh?)L>7CP}UC}Mv#KUdzJCg;~I`AK-`I1$zic%ZmqsiRE80loGA z#H#z~JPx5!RrRsD)67q}sJQ4kGRrTG)4uS2sm{~QwRqM=-1Sz%q9oTjxk*y?b>H*M96ksokq?cnR;Rrsp*o02TFYQGC<`sb?* zg@}l+N&318)kLD+M4C;c63sHy_?3ICaAxD}qWSUwZw_fxRo8a;4y zo%Wl^6fHfTfC~3Fo|;HVwsgX)AOan!f-rD$T;8Sq5&#xYpPOL0)oF5hD~Tu+ToX)- zgEk`0_oV2&!G8Z(Y2tp^>8j*e1w@Ga4O0mWF_CfHa#iaYpwcmb<$RWM;JS)j%Pg_p zpV%3GDQ(wY*lFahqf;e3q!(okb+keOHP)<%_+3_-1Ks%0m59}h8XX@VY$Pfl#d_-_ z)5E<9YDr$(e)3zXkAiq8_xGzPLP3+#w!hL=lTSS{juFY79_VtAc)W>#I*}tlQB+M6BOKgZy+F z>!LMKv6wPrPhn`}vFd+MzqjBwamz&jPLrI9k+rh`?lCm)6KDojD89+@G}P;@M4ljO z{mat5UTif~V}aDMP9hr#vHB*m$-kWn_kKalNEWqa?YOtL&A zr@E7%4$qhdHfmFLo!Zmov0Qr6`o1d_TqU%n7aytKt^Ok__+SoPMo}S z=FEHwK9-sJ*k*duMK5p}>iUqU^lOmza55!w#HpyFMCVZCg6g|!XzR&2;wTzR>ICy# z4ZM+drEY29(9$ZCCFSsF;$(9U|fr> zk6auZpgPfI_i6J$rA`1bmSv5jB&})e|9`aU#-zj2)l5T!7mX?v*s-K7=g^L88RNbM z;20>edcjUtN=;x~IjiTf$^tl(1Oyck`*(xnaZt||%7qBtW}397arrLJ@x-(OnaB2R z<9aGJdLd0oG7>1BLv}63xfY22hypZS*fO=33VsQdO_@~Dk*+h(dXbTFNvt1??23tv z&S+FQ4@qSuSs=0l#LxwMq5t|4>{n&|k61od;l|$8vVqw(*A$!?|AFf_B8E+O@s&&I2JQi zk~CG%s0^JaWG@ey*B}0TQ+ZE%hB2a=5kyPfRHdu>{5G-E7%{w~*X}*LRA>!dd0JHQ zNu~@k$l)Bu<{VZQDVx6`1ZgU9F+K>L`=tLH!McUPQuL(XEo zIew&0UeOe6A*Oaqu8T?&1w_E5=sZOYs^fsmS`tRTX%}JSmj@$SB%X-TqK)xCr8A#q zmq=V_Lu&W-qae(^9(hY2LgfO9ct1iVM;B6#%TR+^>H{K7BzgL{Zzi8@ZZ5O&N_mkT zg@sGdo;egdYZ1xR2qc|VrN8chsC|Il-QSO~&4gFeWDqX-==T(gB50@RVd!cjvAxl0 zLYhDm3yh(<9Nj*VIf{VMDk>b4g!9a#>@5seQ~H5+N1N}m+7tI&#rUi1%F|_G(6Ny& z`#CF37YG)@`J|*09nr3UyV^JT%N0Uqj^;WWq7DS_AWgUmj{yjAHZ1q~M5*VVW#Y(U z1YncPHo^W|>Z@mfG-5^Wu}@P_oiSx6Vl=uskS;c`Z$<8TEkv|D{n#9Nii-FyOS=tJqjgm~qyWgRCrWq8e z+E>XJT-~3P!K@uyV9!|QUMe@PwXi!?b3xGgIy;*Rbn!clYqFX|hExDpER%OD%*0{l=DlR7Vn~ag@b90&`3TKSl(j&+0OVa^kIrQl9+)xV6-Hz8M=95 ze-n)VrR!9 zBkgql5`k6qkvZgEVUAI+DR)gf9HGE;nTf?z-3tc9~$wFMb0hU<@?VU zp%aNUG2PR#@4|6AjrYYx!lEE~?Rv7eznfdMAb$P3E9~GLigEh1js??Ox9H)^IMZT3 zio~y}p$8)YehYTOf~BwX&!s&zGET4U3m;}zY`8dq=xsl<*aOUw^BMz9NV@*b;V-}oHsGp8{{i-11`SfF#5coG*-x> z`y*CWnbOmiObygoDy4sb+1c00Or#0#s--cM3d@2_r@AuI?s(TrOs5*w*c4%%G?vd2 zoM5i7TQG4P^TQ99r_-Af(T@7uQF@X9Lrz;3XTZ-oqPZo3PjqRTN-37!DY=a6fyb@I zdv7&6f+e+J3+77*)Cogk@aW0j7VjBIjip@wllc%&fBCI*c~QL9m6G9+Sy%W_c#ygI zz6dkF(WEHwSMlGHGcuaRcySTCb8zyzRI%Oc6$dWSZ=LI<#CQM(6s@{3$k56&ieRyL zQ8#k2>pxtg@r7-4_4A@vJqq&He@DDpJeESDxlaqD2%kS#wj#)b8L}OYFcMxrAwR2* z(?%^SN4T@}UldqhW*c#EwM-&-N20y2hVn3I)O>$Rh`*0+ zdqTHyW)nyEb16uUg00RF+;ZG2HQRg%(2%`rxTsKfBy|W!!SdyU0YF#?T7hJ6eS{ui za;$1iJSJCWk~<5xa~~$!e}F{<8XC@1HN2nMxl^GsV!HgZFhP!PMa>F7gj6{sL$}1z zlYB>c!O+a5k&;1ncBC%J_HoTT0u3h}k*AAOx>Kv>nc-UO*Y^#(n*xQrHo4JIX59?6 z?|AB)XZKRr#U*(hE)RdeV}|Z@x`~>H8HY9mU&w~HS9n82{HYQyd|GEeI~?R~_mrM# zZ+obBhU2m98HnkHUO8q*Ha zDvx$@meFEKZLU2h5{?oYqbve~B-o73HZA*G*PW8sA3oOOY+8&ocGVh#X1n+z&j07v zQRreAzuc)m|+6B_+#4 zrB70U8VnA07Nj_6-1LcL%Fg+&`_BOagVk zMnlcXfTNnOvT5QEa+qDAhz2F4w+vR7&!=UK*P%5Bb)`Toy~$3F$^(KEUamOSMYen` zZf0p-%2%>&y47rRRAg_ous@hf>)E2DzuZ}xk54tjY@4YH4gzP#Kmw&7m)d&X*wc~j zEdt7mdu2AP1jF^B9RoC>C7(>|-2#P7RO=Ky8|yfL2x&B5R^stDYJXLw-@jlo zfPl`XnqnO^i2FgF?>X}rOES~ zdKYU+@Z}XA7(_jgdmqmfHi36@@KJ;xT;pi%U_(5PwwRJO#5)U_+Hne)Q>#&#YU*q5 z7^&;DG8CX6uSz87{FK!3SzRbQ1sP-gt+D=d2}S9^#K321%8>+fQ5Qvyc>M;7|8k}0 zy2?>D?|Mnp1@LdtS0B~Z4`qm^e-lrF@J>amBr>#R%s4OvMJ!y3C%5I_FfzH9FRehH z0^vPuqp;d?VAPrL(K2)Z{_oee7j;V5H+em}XYP}7^Qkgw9ZS!nh#qqC>3O@I;2$CB zxQ-tOWzj|@eKi`4lVyw5edP3Kvu5ELbju&#dL0kx6cUIPpawAwi+P`)FdW-M@mEa| zewlqAyc%3zVVb(C+#=iWL5-%;awXn?Jzfcwh^C%%;RHseU%5yQ^n$C@k~v+Z*uo@~ zGk;px0%XTW8EMDPXMJS9#lEZAl8Hxsx&6u^m__XzH6ftTnwOcYtNb!4OmNe&HvA!U z0KHkpa+3W($s+Gug2{9`TJn)@)wr-)1zj89;nMmR z&v5lMBY{d8SNYzF-J?&`CA1whOI%?xz!@eQ>-*d}aG(xzo|ZjeD1!-+JFXSs;;JMh zFFpPz8PMBMFsxxUFb=fGj#y;6xlNg8tZfz|L;L_ke26QjkeB}wzp(K7P^UO9_>o=I zxm#^%kYvc;=SRJE>r`pXSAhbr>BlNY2mR1Mb9vku?%g!;AWEHOfpm8p{;1Fq!=RGW zxaF;6Le7>J(m2Mlg}I5Q+=t073$@?XS=KTRw)bweshYzzwxwOgSvsw|{y?Yyg+Xbs z7^yiXivv{(gy&txtA_f09Tn-ZT$IAltkDCwgIx!$6oLX-LyF%iP}w^iufB$@b84s=fae*IPKy9^0wtzHpZ>jdTB&qm;B^ z(uG1PI%U(7#~_`|k!o#FSxJ}~x?Kx}F_oX-atFLNSWv@Mj& z*a$el)}t!(uFkNk{)hY&516?_z!bZB-OELL>xo3tX-}bCX6-J(P_oq6qQQfgjX$g* zaCyyd8`UPj)31igc51V8x0MPh^}>^B;0br0zP#Bh8i}AM>PZe!-@$0W6Pxrke|0;s z*So1JQWWMyGsV0j{Ft(67ll*i`5bqT_Y03jYx)Sj)1~1)e!KHUmZWf3!g3a`=+XDo zL2->HdxX1f>5-7h%3*tHXOF#3!`xDd;xF&_PDdLEyMu#_Fp)#iS?$;y1440LjZJg% zl)M9BFDu!s9{H@fqRLyP*6(%hg)?HuI%SH?li(a)(O+cNZY}?d2>2G^RzY3;9{Ar& zac*RLB_qdjQx|W*D8D^xs&<+?w7_``%6r}(mjss()gGHjux0uoEoqp}Xs$K;5z};W z;Bg5|<;N|-axejM&HjThI|LdD z1QYl08EUygjiTi;HQsZUs`fGFv(Ort=PS9bzEXtU6SjzicPB`I!TL}ySn&;9h;p6U zwi{~d78ON-ET$G*CI_IF%F+HMoR&0I`57Q*`tz_9DL599P8Apqbd_tf{!bi`(d{G{ zht4#b2YKdFqguEKMfz$N1FX0e~|QqH|H%Y0F>cDl?T zfUL;Z6-Q(YoxU3KJE+y)x|f~gmMrY8j}!I=z+yla@Yo3ijABFNR7Bp_FZ&|Y$5+NP z-(HTV1LV)F&}B?B*_6VBZuMZT&zUoS5EDhb3A;8_XO=~;^H_nWUiPh-8KA43w*iiN z*A4d}IB^j_D6gForI8l$$VHBrxtPZ3fv z;qnsthsWPm^Qf;b2ArIYp&V4!i0wcw_OPSpD*P_r2XHv(ntn!G<%@8NFh83lDf@ zyht$2V?O=j0e_o+@`wkJeu)c-paY(jz z03d>+z~5VZjcJY6_YWH5`s9+$j^)Up{Fw?Ta{Eb*X7IXDy(inIb}QXDM(3Nk&iwzr zm*ivLw$r(i)Jyb2NfG*CL}4Wa2=mU8<}syNjovChAvw?L5yNsACpg!yd-ZY0e@Y_0 zJU-D_Zi1NTT}CA!sTBmp=?vFFzT2dAEuJ99KHd_BX8wNzR&Y!!sG=YiCogrn;S4HD zc=?!1Kd|4X!uk{)dA?1^&_Sv&Ho1U6aen~T2P^R6J_Vz5f#N=IjY_e6My$N?yxA3P zY_;DY9#KZbzolhaksdzQYOZaC#6eVJ75f^LU<&RLP_@S>29*MFF|}m!0uG6B_pc)c z_M~|Y*XdLDlzi8dHjoeLh8Ogb3sccS{HSe0YDpTCze@Ifn>^azRx_#OsQ(y`&hxzj z^dwK2h}btyT3}p{w z65QRL0>$0k3GPzd-QC^YZ_fGd`JLo_o+NuOnKf&sHb3WR_=-|tT1Jg%iN&CvD; zakck+%U@k(zPIjTS~D0_u3D-}CJbyK`@f@w{;O3~f3r9E7gj?2jYExWD~g+&fm27f3?pqrsF6`6Q9AxE^EErU~#4@k2Gr6kG|Y)3E~SzbT9T zd?qQd1O*+E8@?;#QGu+wdoDhyT6aW3)-|m28afI$ijHQkE+Bd|N#U#IM zYR4sJs1RD&*^}+S9&G#oFkwCoSDCoOmmRXs4so&!8ey#nN<|Y0^kMW&#{_fp^mTq0 z1gU2thQzRh`%->k<|*Dei1n-GN3r5p8<4{(*9eyg$`9JM zX2Iw7&irWyb`eu+-K5(eJB76P7N1z4uVzv&DB!t+Q5Pf_tW5?tkq5soEn6cPsNakEY<8% zdz21XLwvNwIn{GBoVo7to@VK_l3(f_!HXK3^+a+dVaH zbtpU~8j=}uUCFDL1a2?t6aiLaB_|KoBUC*9KR*JpcS%IYl_2rhd=671W=eB)Y zHXwIia{KZ=8o#LTH&%dFRL-;cQU2&GEMcDLjAW#(3q=K!>>gEn$c7XOY1T1#OnZUV zvvM_%#@)MH6xc7xT4!pR#qO3~6sCDwR+k{Za%190H6`3Fr~*9P7MDP@A3!5zF5Reg z=2P>z$1*6xduMH%2L485w%$FVm%*5+WZ)w3M($f;ap!Fd)|{)azI@O%#e$*TE9p-R z0aI6GVISLU6D)cKwN}!PJJQ#$b95>>mD$x1N#Vtc+Kx+3CTEZnJ_nNR3<=<`c`NFF z_WJcCyG478S22l$Yz+VXSmwvz@YNfl0^^^ zt^OcIs)4#zs>C-m+!sKVKV?=2W^fyIFe{Y{7R^_Ro5m=!6eS zp>~B_g$1mUjX5bWi18d4q%^Db=?Aobo@+PJ@Z})wV2&NNcPSw3ThZuOm>P^P-D;ZK zPR;Nn3r0pE59LAFfT=w+i3vax=3;!77Kd5%f)#}m;_KnyeiwfC5Af{8@XkpVQ0DE? z$ZoC7wm0+sQP1W;-)?S?Bxk;l%#^K_TBb_rE;qgaO$V$TV`qV;$&~SDl6a>>a6z=g z5^mib9Znml4bLZ3I*d(T4K;E;r$R2+D7@Ua(E8&FS}>?5EdCGi?HHakd-tbkL-rT$ zABpfg?vTuJdt~GCH&`@@T+4KbpoE|Q#4;o9wg@ae1~b4#1yi=@c#1oWs{&~V|0d7-EqjQ# zH-{Rf68*^xGeX(WmJ*~9P zq10_Mi2etQgR9&FNPIdimjQj-($6iry7<0+1EBmJSI*@`8i1Ov71-ADrhFVSqr$M( zPZixPIFnRaqheqwUzJkG|0pefDu16)q+9?mTe9Vep71-iW12XBu>l!a1;F?tXM7(dFG`o&s<8u4m@@9z?xnLWq$hF?bN6TDm!)&8mG1gqql84VdV5oa0KKcUQ883=$F=O*o zP7{Ux&XJe?DYPQx0-N>URdhgv_70jt;EJ#$Tg^)!Y{`J`1h{$$H=EV~B_nJ!K1_DophxtGc#Cux#^p(@Q1FgC*?S2I6%Om01|+5%GDp%E;`$ z#3zI6D0!q>$8sT8@4O5y;gZE-IhK?e4~nb5Hd5damuO*Ui}%W?js%?VOMc2f{AHfh z{B9t@MBfl)9@4djWGtK7X9O_z<$Pu4*eh1ihP)Q@!Mz}}=&!W|9S&EYw9^sE?!>8JQ4N?Mhb0k@Oj7 z&3E{o8k}aeod$b*`WtR>9Jyhw#@`|6cpLdT<}v9-q@Q2)E(q)Ev*ncI(U**eexYH2 zHkL;I3tR2iQ5%CSnApV^VtgoE=tlQlQEzM{VM-u{#t(%Ko>SEx@0TJtJSGu^gB)=6 z0$det%8ztNPG(ovx`Jz^j$w~gBXzQgjpUGz~vZ}U&_JR z67(6qWlj3siX=-Fth{!QwMQU_Us7=>E-nZ0w?7p*Srl3;PDP+DD;&~`%1rfZ%PA2b z*#(rVW!bZ^-(Xs50(8a);1(2{#kd-*e!ngyCC*1BBTpV)AID^|Q z|FS`12jtuH{DyOKp!Oo?N=F%(2^>z>MSi~j1H@EcH2A|ygtC8HirG(l?U7P^x_6oT z#w{~+tII+ae7e|1H$TF8Pp`#0vPD`IVPg5#@d=N4Si?L*(tt)0&8%5(J{%B?KQ=~} zf-A31BDu8IK}40sMT;@mFBldB4iI+*`5HuT9AIT$V!amA@e}90LO$6NCdAkv#A~GY zZdilAiNVFreApL~84%Z~G?r9Wikn@#y*KFU%gsLJV?qbp zy#wGpXjAGAhDY(VOavcJ=UUkT!FbJgctk*$`&SrQ)$@m zCFZhK95aSwL?gZGGS%AEeI-8N-LF0n8uK@c4Q>lU5kFn$6s7J=p!YY2b#ZsTL)Io) z3j&(z=Q&QqF84xRfHF>dWEvX(Qz~vjWa_}}j9fbp-G=`0K)OwvTkkj)gCN-cbE?k z+!A2=?R-#mrsp^e89#>jctDA7)a3wrtW&m*d@ zP54M1#)87iF$$h!g<6XLM36%$0NTjg(8zvIB(lzvOpgBns~As(jwquNK96eOksU-A z)966r$qw_obl-BBuG1ZiRo*((nl9J&L_4uQk64YQ%Jk3vA>3z`m9byg8!JjBJjBlj zk1W9Ci69JuZ#3;vt3w4Wf>y~S5iIQuBO}YhsQB;>*-60~T5POG96VG*xEoKhwn5Ng zlOc!VEC%qx*hLlWJkaBMLA6Caunfkv8Law4j9AxkPOZZ-uxJ*F%5%c~jx=W`$#FWa zw=@$yBrp-!#9q)KT{_2~iKZh#E0jW+27xsC&-}JQrui zk4OY0EJIot50Sg!P80>PR7Hzd%u865z|Va3``Krv*$dmD$XlF8AfJRGpW-iSOP;6_ z#7g^+5fO_pwZH-KL-fDAGa%}A)3^<5MvZ)<5qE+i@ z)BIQq-&3F&m5!jnsj-l-LHNoP`L%{=UM$J55KB3p@DWAJoqtH#Ur8S;VUyBu94@2Q zO|VhkMfseTzK6(8?ac9ZF7_d+3waWEl3m~KTqTyvpf=+4kBPChSG5AuOs}`|nau4@ zls==o8bhAqp@QX!cv?{a~u5mc?JGsI3nI!oe%!K`S__RFn{tD{@2gn9Q9^`LN%1kK9+%7|*b4al*T$GnT#o8nuR9?z61%}~Ns zo4{8kFxgG(SL_#moSWmDg28i8)c%8lv5&RS#F#>nNAU@7ul+q|d4c zQs?ht-e1*wbz{Qn9TzQ?uT$LRU*xW$;Y$C_EN^=(F+2Ph&%F0_7juvL{zr|~Cy~p7 z1@+jd=G0d_``<7|V&-rT=BKkW(l5AEaY!6G?b|b{FsvZwUchg!8in~crVnK)6MB5) zhLXnExpI=68d=djFx!dWoCyNkw)y&>2>wjFB8k`NKo_mxe$$P}LX-bcyW{reLd>+F zQk;w&=>-R8YNqsqj%@O4v*sHmWYZY2GrQWQ+ z?E^>9xbDJZlBM@>)${&J>~{39dTHLlA-`SlB|!UzKh@nRbP>jFwImT6>}I&>9}0Hm zn;19Q)lt3z&j?^fYW44cWW!6bV0p^BBCAZ5>}FHXerSR*%?OP46uC#j<-cA2QF5h5u?d`-kfjhrL4*^`P8k-y3tvmYu8 z`(_E@-WgR)xbdzk{LW%V_|L&7-)9N1ajZRx;XMJ2$d>%w6CRm%h_2E?VCSUwsd$Of z4$@|V`l34=mlg*qMP9V`vxK2y{@WV;Mcp6t1I`d!JmQpC)wO4kfu3enH%U^%x<$YW zk#$*?``Q9q17Gl_FC>aZX;|4r^!7|dfCaTpsaY~TwGe`6JI2D^7!gm|-vM6`B&b2y zx9(y)ST8Api5%T_*BPz7&$!$Gt^uU|gqAw{b5!>~q}n?mjG`O703EZO-|cbEh~LMm zufNF-kR+LofGMGK{{&TI0I-wT3kG3@bg{$P8pE*liC}!70ud+(y`Pq%da!|p6T;9t z)es-wp4S*yLXO_bbHfd@$8KeBa8I&#FO6H1qQ&FLaI@lEsW0Q>aP!-mwvzVVZ^-C@ zMjfC`r>=+atrOQ|M_RS|^d8W4r5~{_hI>&^FMzxXGvYEbGpX4GZ;+tk$ z!}w-yeO16uZz-}of!FUXc#YjGfqsKI3vpB?Fz@F9dut91oL8%e)ujG>$W6Aj#nc3%E#xj zCjvxEG%fIH!S5%Tsj2C3EkzxD{um1@&)zsVsyAsAS>0UP#36AW^+l5o;24@xYOsW- zrdo*NWtDbqc)~nlwfpd~vXJYfY7=){Fz`B`VmOa#24F-M&^1yH{?WAAu5~$Q=satM zk1tYON9_B)j)uDw3Q}q_ENXPLZr9V1E z?y717(bP<6`rF+-bhN6~WG2DFfKokcDsef`6SWs-8jcWXS$L4kNCpPAT0J+E_(Lp|hdNh;gLj|gBFyZZ?<^<~T3 zNwv1uv&{PoO2F?hp+kms;X)nncfG;Qb}yXXF-HcT*EiZ6)Xr`}ZpMmeP6zAom_%FX ze1?E3Wg2L12WA8^0eCzk`cTm;V=qCYj(-m)q_w|f&5v(Im%bBMiCX4S;WR4yr#zu0 z?NaB!;0Cu0h9p3#r)g-BQ8k_b?N4OcT@OTXa$du*amz`WkZEDnxdmcPlHqA z?{+LVI!EWWDu~o?_kgj1fq$o((STcMcLDLl~*}Jd&l#j`|f*`Y*_BZ&(C~+uZ~e8iTX-oUT8@Jr&wj;;hOsFj=wQF|m^y=r3`$ zk0j<|B7XEn%#otYcSOD_Ui0l`2~4DS*c)Psd}974ed7vdH#v`tkYw%3EdCI*Lnd3t z_&3%w*1u3Q)PXaViWbnrldNyDZz2>Dbu`n0AQT@10sE(ckl%2usq;45FjY$>a;+=Y zIq0n-a*Cj^@qbu=b-C!;cCVQ4>7_`9SU?sl)hL10oR^}}uSC8ceOU_%Hhm+nD#-II z{4uAVIKbkD-KpB=36EsPh%u+6Vw7Ue$nToz+L;|p^?Z}m9?f(AK1xZhwD#j~} zh$g45MfO%e@h$G?7l|;mahhVN|1pH7owq5QvV-?}+lB?0C|i;8f)1L30H%Z)o@QA- zHd{vy%>MhvU)&Bv^qW4S^ga$2yPm5b==v}7jmJZO=k#Z)(Dw=b>U=C>^gTV@d~sc` zU9&XV7gDX(Y`uzaaWw17)x;G~%z2QxxjKoIi^{a?tJyZHNV>0%^GkN-(pu0JgP3*h z9w2#5p6#SjDaC@@cZ~)cvklwQnuf} zI1-P0dk#KSYHWC|j718MyIxKsbaHMVRQnv_e_nRgc6IG^cP?MH-wXgJ?KYr;<-c0= z@ZjMIV0FPz(;oAqsru^!y`!{no+WCS6rF}P1HdXfF!om&h9EiD>nRPa4#ybGBD@9@ zO%J{~g)%B1d>{o`s9s2jNdpUlJ`mP=p7ZP0{ABHqzm~OgoK%%nOp*xFnp^mdJ(4g{ zSrdP)oHFTZb1=USa2W_iAw03zwG(ORjd|MhZG~d?Cx2SwrCV$*&fTe6p zk+o=ASy_PpxryCO=<=FH)`+H>{(fyA;&MskiRRQ^d7XY#KmS@-451417?fc{%vdRE zwzOvMCQ2qi6zR?+GpkyFR}XK9ND?vS3&m83A*S3zmjxiKqEJ8}SrN^jO8I!Y)wU3D zW_r-)nRc@6-~6uCz5@~17v|5ZvUBPoK-gn8GXH+0GUU{M|HF&8lmZ+jsGx^y#fA|U zkg_=q0aVoS#`F}Oj!+?)=FIlv7#G?lDEykhK7^dj)Xy&d2Zb+$&J?1JI4eT|8q>N^ z`5FajL}FyqxkQ*RflE5dBfz&77v}+FtZeLQ`p9R(DT16*Wlvq0>|uib`oK&tSp@9> zQ?y1B$)R8hHOJx;MdIRh(m+Jw2cMoS3!~sU&^f!$BYkVB<3BgHeR9KV+B7ah1?0<78 zG_>qNs|`Rs&9Y2Vvx}z@FPzNZ|I`ZY_jZvgh z`3A^LTffOM%U1(AD{k*4M;H!RzA54sHj|Tm>0TNn?#0y*?IrF*<*qB_HGn3wp&Bz` zKC7d7F=|QC=avkECjrI17u?UcmsC{fXtX_8ZXo!%@*7jC>u&(z_I+#XIdi`K6=9^c zRU_JmUQ;zSr3fRgsPH4aJf`ujq5f2uV(}*hTjX|mGy6s zxSi~SrL)C)wK4BtUyjR1FaRfDbhT;n!>6b9Fxv;`TP`XK^v^FRn6wo|-CKL_SPR#j z_J+jfQTAg`*1z}(o)ssDhz6n5q4F_>q6esVLj9|N0GJt^S9dlSz*4nxVj3F_3>ECJ zYa=B6!ED}Gvgm&qtGj55)6t4~mIT!-f&BTsR?qCzp#gjRYud#C5}m|p+JZR!3!A1;s&Q!n;G6IHg0ZRAHg4P6;?D}EmD5r>;gz89(0Ha>G6TBD(9Sg)Q)(6P=5KbETY|> z9OukQgh)h05p9z^?mq+5hFn(TX7i&0BoW}tNjgpc2yN>luko7{M8fNZ9 zfeoMARFdP|HOtW~q0F`9C@B}vue^z4;a(HgW6TH8@yjMOje~1g`k5xgvcC7M-Q3 zXllKRN=9#`lvSa0XCr(fHJ`&IY)>0$yJX-X>${K}hS@-UDzE2>s!5RW1DZ`uusqaV z%f^&ba-Dth1f;!9Zdb#`1R~^ixtJ~18;s6$Jzwecer*hCUD?YIghe@>Z>xq_Z+paJ zaoXz+razw_UV(Z=EOPamM9!Gna9mKbfbE!=t4!?0CJqoG+g&iaLEfU#IUU zPu7(AY2TF9@p4Znucy}q9Bf?obg{&oZve)~>a^Kx)HH%X3NbtHaL8B`E43zMTX?%u zAjidvb*~Wqrx1pNYo#BIV{FBp@JBV0el8q_b?i{{0R~6N>ONzL20sYGhoga+@ z1F%ibee7{zWsOCqyDw}=NJyG`N#G)7*0;NpuZ@Hm4+skf+favp;-*gx%?>WJ0Gk+;qKB#kh1hPv_8P z9TtT^Rl)t&oUxIRw=_^G{$Ipf{rn~-UQ4~;wr}fYIOvB(7X|KyUhR8a(s0uv^kMr?LrDsf~ z7JK;49WhgZXN>Oa!?|vk3mfNg#Jr1y&;A2)G97=fwZ;j2kq#(_;Wj6o&>m4!9zSnKY$_dR+Qa^SRllK?o zvxWRa|98(O>s4*%tDFoYriD_ai`9Cq^(y5Kvplj6Z6of>JwEw~(_k?UR6&1w8Y~Jfsn=p_11-wgCi-KrOTbk?eL9Er zQio&uD2PF)H4+(Y{Mcx%#m>&gMge{z_$MLiSS9Dey}CFmt4WN7A``9Enh4GJNU|31dkyEwnP#}ygvw^p&>{zQ zaO|0XvoQ_XzNSai(2Kf(NkXwYR@NRHh4G+L3)PAY?I~o6uPmCLHkKil}S% zhU-Z>UxlHkjb~#-3p7`o?nPF;kh@uEw08gW7jijY)71bmx+`jHb2}}G3&u47RS6V{ zh^iCs&ervs2SZ+7&R5wlHyZdXdN#aaoz548{E!K`k?^@1LAeIKttuB^X$g{#Ftpe! z(cP~8Du8b%)+($aYmQtOBu|qypWBt6k6BEXxA5R*jZ5NH>rS>%XiQaU*>_%SW<*@} zvQm)!ZU|f6tUn_L{3E}Ya{eAb7rr| zgbP)=Sy>rNlE(t(>fojhhKSS6t$0F5Bvz#7zPPWdf6E}hzun|~C2?LwnCzGalgR#R z3oVZyg{T4*3FP`HyUlyeBQho;BC-LC35~8rcDhh;v{ZUpvz=dtOsLu5V1E-=a_jRB zcYngFzx$%%>2f(ojFx&fT`srEZqnaR5kRvuT!aJ+qFun8DZcdEiJLjs>Tld<-`2W= z3q{1&QB&%0J{1kC77AZkssn~nRmdafr6Q}UTB$eH78R|G7)VHz9`y%2163KT6?X}d z^CCiG@Bl~jUoX9dNWHED$mFkv!fM1iMlm5`2HEq$`&cQ2>h^}KG}r0R2!whS^$x}9 zq&`-+(VN+-wGP%l$@`be6ovs#4k{g*#FNaitX8?iE>ed^%8rNLJ!xZgBDKX7_HL@n zXHOEEp&UNEF=T%tUVn;t+QM=GJuY|SG#q}Wex00xA{thz`dfU{^tyeqD|p})aeSb& zkzv8cp#wlk(JAX_(Nv`@=y ziq=vDD~b%bYAqv&0T^VU8p$z;2Q}#;XqO?ReFzFUpOF&xG%VVXMKZ#J|CaFXe71{A z*Q>|-MH#+QWH}8R?pSHg$)Kb@h`T*BVg08I44r8-sjSWY{z`ey!0AG`rEk&GV_Ce| zv*G|(tiknLDl}jn8)bP{j{CeXeu9dDL3RUxCHe8RwQl$>wI#*arCY&+!1SJi-oF_q za+~!w8yJqyb)3K&!($^VVUb=y)Ayy%C}xcoD=}_b$0}9Z;rSk%<1os{ejJ2^yr}7U zJmjNnqF1-t^ItzfIrO70E6ECYZsi!ry&w0CR<$d5SuWQG168Y0`|+3^&Yr=Wgq_}@ zG3 zu=064WU)AV+zN>*R^-V69|X`KxuEQbFBKsn60X_UN#EMIoK8~_;y z;zt_VA78xGDCCpB!%(p#s zw)uKrREz@TVhK2%YHqT4Y*#xx#r`EJ7y)$cwpGq~oTPOzX}c!*pReBJ4k&NI;9WHD zZUq9d`_;s)sm^khcW7Ce&@I0Qu`;8qYzMrG{f3B|ESKSnlCK9udmD8GK-&Hwfprr5 zl?r9;HT~4~CH=ee@L6xmCGX>0@33YU9nEc9s6Bdg^VJM+wVoQi1rs3+4*BeLmh3U2 zP%)%np+c%E(__xQED3qtdvMalW;4&^^9Y2_*m~D`<3f0cpY<~PK8nSNS{;qc@tlFM zhoZ1%H_8$G-XHVn_;`^yl|I`II-`OZUt1TkqX(>*0%sF4W}P$jSjLDXBK8UR4(2Z zU8jc}KF3-Vg*WN+0&bs`W?9X8O{GisEf)v2Tas>X+ij|JT2#7NQdb<_`>T1{ww*81 zGbqIb&TCwUdd#oq$=SXSu|W#E0NIYCoAisfaTFSwhj2h~hJ8;sAX{Zc6d96Mjm`tC zrxdv!OD$c?_WDpK$2TXh=X#P_C#;NmsTd{zLVpXqd>3fS1<0iG z->!`kwLYrVF?8rIe8OEK;_+(Ff#b`EO;3BvOQsmH>j?;u@v!;%+%$~$=*;nXzA`Tb zqtHJX&oQ^S$oJm&ek=UWBUtfI&*7OKoSB+Wn*=1Y!kj`}yQ7x7&F$XUfm3*Hvizep!_pAJEZfD37nIjuHhf&+$+sLzl)G z;fPPY?`A&AI>_hz+Ymt8X*o(vR)D!jWj^%cWgI1*Su;p!f>Y0F-uE-Kx&W|P2$YMx z-Q!Ev_Nl9p<9Dex#|Y#2Ue)@s5StI6WKgKzI!TlwYJXJA+lYGvsL-(M%cj^TJssA` z@z zsaGxZIQ*ZIR~lX0JEOK~4ttkQl#;T=$OFt7!7jEv_=#MgUe&l4%WxhVb5{se2 zm`|jl1gvaPsGPTKhI$K4RFZm$OsPl0w;W}!q&20UYey>?ngwZj3nHvq%f6-x%&uC* zwbUYppfu~;C%*FTfiFVe$OmoUmg=|?lYG~iY_MyraxxS4*UVyz46$GrDW{nSzZe!p zom=pp$m+Y&SV9Fu_7QhuMLMvx4LVYEp~PBTlaz9Sx&x8|fqX!`B5;>2{Z?fgJ4xE! zKcP=}D9GB+ZK$L@#y`^jaak*qGB|Ju^=(F9*TbB;n6=Jb+lD z$D#s->M%0nQEICcZz!Z5~2vsrqo~XxiLDkpAL3~*x%#2MTAi7y3{BA0woMO%xEVg;0 zLjIr}F<+d{8JYzBI7_#y6?jPthMMY-=uR@M)6!0IO#Gvd=zKFKqkmF)|3zsW6yZXL z_-jV-az>!y#4RUslF_s&=yL;Fryy3OnGW1pvw>#JM{3bErx#i*|D z{d0dLe`$3j19RA+m^9x@LHwk4`+tzr4J@g1qX8M-`#Z-;zDpL+K~}>4&S(`-Nz-GI zu*rI}u>sg*V`#Qh;mEjBV=!Koxhj%jc>MXMkqq`;pPzT1@a}72q^IY4U5nK9_z0K5 zUtC;JMSxo=cKwWGY%_hV(vjo;=)Tb5>EGz(I@IylpXb>0Tx0}V_E~cB+}GEIA`w%# znD@~bXsq7k$M8Ha?aD28dXEkgkEz}gzT@GJ|z$OLS>1G zD%58t!i>%EbhKRjxl|;xS~t;lm$GXbw#Qzj_5K7-Q;*r4Mn&pj*)C(@mmTkAPYFv? ztuHUtF6NuZ#{eSDHdFV*iS;64$C+zDrccSs)Y{{Qbo~K5Ae)rG)@k^yrEZhkPXDuH zdUvvIi-oxrKF)6@|4_#+cnQ+uJ<%+gUT3Pv`WZAh&S^2Aktd(+b=O&Cr6+gItT#?P zAje%>Rdp~Odb!bRA+!kzCC9hx1v+DB9A~#thnglmbpWBvoA|z`5p{TEyBxq6 zp2{jWn47Gp)iJcbbhzs3v>N-Xi(+e;gHOSCwyT>+IZ9`?)40>V-=A%^UA6ifp@xno zYfsdGOos&#HEqVEiI%uDrD$H$*^ma;wU7LhvJ;8;81C&o*6Q{bsiyb6Hp$3dKS80$ z2q~Y}QR1);tZBR17{2Fny-!cjZ|_6f4>{#U@W5=0_i!imQ1=C#lfU;{F6Q=9sI94k z!2lza%?`)KMsUlL>G0xiNNYHI;N0eVA%E+|2~OR@^I)$=`HBVmpYT!_I+>o^iSfnK zo@X(SbXvS}4eqiT`PhzpOk#hcwqle~HrdmFAthxiMXS)bRhU^-{p zMUidT`1Iz-iLY=3xQCug?e>nJu1dXfW^(X@rQGsjgQe!#u15qLAG!-s}|8S=xXg&`7?**G>Lkvwuij0nXOLlCm?|e z&4zldrmWeRO#2aw!}-O9&f5j4;R^m%23RKZqaiT3n#@96U(Fg%Vn=sq`u?L5&fl&D zTgPXY44hUicGGfuMYDhIS_zIOj$&7+H;Q{TTc_6oZ9hUvg2ag4*1>>;=lu!C?V1!r z+rg;o#|zwQlg&n*S?H274!6@>I?Z})K4lnZKQe=-DvR|{&9q_suli;;=aYJLS-#x> z7~(F*2l?`HWSNwoGFJRLUdu>%A@OfvzR@^rx&;(Ti=(5f#J)4QfC)~m3O(Pek90Xc zkC>&uPY0Xcw##wulf2izl$2IbbjDiGLSha&>%5M=rl(09aef*HU43_bdx3j>xWE&* z7N1_9$aWqtGl{(Jz#>$w)c)x{GTl!>F?{ko#H`Ts3dU}do=`+CkV6oO2|KL&cjO%m z1-8yPbk*Ch$Uq(5XYj?5)_(DFJdgc{M3{`uXLI)vKwF(ImwoVI*3hue`RpcD(|K$d zt^Sz#e18O6JnCenXwl(mdcS)0 z1vDZ>r}@lec0F%z@bLt91%!EuB=(dyq^vOd}HrH8iNIo2Q#5?)uB8 zayMV8rym9)_Yvy}i+b)JJpjWB>NKY3l36od@`HE@VK0~EGg9q4bk#$XPG>qUVV!Kb+mW)J)4OEfy}N}z6hZHLd>A#n zbh%5BFwADeL#0LjMd~#z6!FklbJP2^ydU~!uWBdX{+RPn9@-kw{;^~ertd;Mol2?b z5897|;{7sJk`?~X`d^iq$If;UiS=sNWL67V2_rw2`YLKD$}re_9brRC-sz)yGBbxY zC3gn#{-k0u53Q$lLlu88LqS#krTEa-jAog}8sEAx_Bqgn`1xTq29Vx@6z%s;E?)+v z9S<>_8yVpt+l4D9WhI(zNyA?Dv0jbLbEwGWE&O!ovVz^QqiD3ib@UIKK z^h6Qstq7bQE3pOX;COGY*1?F9Wmwm>#q>WpFu3>g{AjoB@uX|jaZ zjwD>LB4v!%Gf5r1u~aG$F^?6y^Lp8_T%{9g6wTxQaK_DaT+dmNzCbHZsgq&_N(G%j@*7hS%qM!dcE1{$^Ijr1sfAn;B#!ZT$Ntw(x5og zT~|n`mkedYnTbD~PIc_`^rWi!ASpb)*oV`#(QR^SJ72glnJx{$Z(p;$bz05eWdFV* zqyje&iNbds^*z3cYR||fit65(ZKoPk3JT{CF;SbGm%u9RM_*S`RHS);5S|v!7)_=t zTk84@(fT-eikI`f0l|@Mz@SCmT(;kjcwV++iPapY0*J7H@5Na@6YX*bOn+^BFSB%F z_`KxeU#e1seE>#KRBN3=VxT+>LIP2;BsY`As`KjhJEz~($yu+nY9A+;X?XbAo(I#$ z1m4@rgCekf+&+raVlq5co{zdQ{7%VB%mF1LG>z8lb1hQIv$6u0yDgUycntK^e9sA~ ztBp2bqrcZ(5mCy801>Th4se#wjiV9(Y4g`PgKGYV1z3#KH*I>kKeSa-s65SO6P>Nd zV4SSU@%v=+2-O(y+%oI23AwvJzdyctJXR6zNbKK0eF>`_<2=t0Si46E(Xp8HA%P~f%0wtboIR-_vPA7U?uiEvSB0p0&4nc_K^PL<0#$B#TI)N0A?;o4%`a{ zhRUUI-shuic)#zPeV09312tG{JRdQmbGw`oH0Y1{x2By8ez>U#3DxgBFW@sq+-rao z7C}bIBO~vi2TXchx`pV|JjU#8M*h#IKI+*%%@a8vJFWWdK7)%_TiFaJ<4?%II72WT zF7Pqv*=W10Ov*oDa34`v*k1^afQ7k2qUXykO6BwJ#FOs(g@AajbW<4NHfrfJ8ksEK z_gFaJE{EqI=V%ZzQ+}mfetUoZC^_X}XM0+iI0c4Aj>Are89J*}G>YNVNHE$FijG+- z=qeZyw|~0475>t;vfS`IWc0W@Q<*8MFD+tr2U^QCsM|>=B+@2_!OUuPUCR|`a-CLfeiSrT6*YE(M?Qk6AKf3`;!_3! zl+QxH*M_pao;KQJOc82q?r&T-J&wVw9O10euuS-%vG#` zM{#6TBDd$~!NqEgR4;3NmHXVj(fP&2-Rg{+5IKIg$NfX%K;A(0{jSHyCW=SQ4d2K4 z{|t7J*g^0#?51lR)ta5^XUyOslhf|Yk#6u3T-xXL*>!yMch8yO4ZkI`@2_yv9o@fD z9UQ&5p@Ar5H*{#(b7j1p%8t+!>Ep~nOm8s@b z)dajg&n3h>cN5J*5DEFb_S8YHiW&xsx*iUBji)=+pWpGNS`S2;Fkj*Fl5~CdOZSHH ztNCZaH6=Xj#CIMEpc=gtQBp#}+I2edPW6}c&uLGi0a+qGmpBrw8{s~(6_5K7irj#& z5&ZW{-n2=Kb8-{hmp20;FRzy`?zM{)dEdF(FOqD*xp#@{vV)<~@>Hr@ti$fysKdk$~ zZ@a0jxgu&sY_r_{0j@^HHMaw~Y$IHC-IkGpkrD;sa}vWH_C9_Uy)V?!A>-_T=5OE zX0ZNZ+sz*ejd7`wifkba1ckwrcC{wDSY9m9{ zJY@<#Kt6yy?fCI$`~q3Og3fMFYYaW9p7EYOLurgap!aAUV{h>5LKN98glg+g6u%jm z1crYj#S%rs7+*!ljW(5f5ce=xd%Yr3C-8IL{3cLbrb_VTE4~c$1*W!>9_dFj15yme zW|$BRLU?Pi^96_8Z%mlG|Ryetp6DT$<& z5QkMJI|p%?8tIJtZKnSOvbhaLSJEakVzyC-+hSHp z(S`lG6N1qx_y@Zmn;3I3UFQm(SzwN00l=^0={8uFx|Ql}LVh9sgN0f?D3?N~S|zob z5Iz~^TczD;oUNXyM8h@wMP*Xo0Q9HUWqqVsnfD=es%f)Z27jTy+(B`@N(=wpMI^fu zDn20m-W7!DH1mg*+vy0yB)PXNQL4#i8Cui3EU3r$%X~}{bA~FZV(2$kZ2{n76-MN% zHZ^-}-qPYRnI&0tz3r#}4xdyqgS=)`xOW9zCRFvP_g;7#K zx?2H38tE1(Y3c5g?oMeKLb^e^W9aUoyCtP#sG*zp`MqoX`3Q?IaORwI?|tokxQxNI zrAm|Uo73ed=A~}mqnQ63HFPTH)!iIn)#{fjqcXg z^xaXyUvpDaQ%g0b(>`Feg*HWhSb@b^r!k&RtB0SMZ_vUl+P;4(S*e2g^zhACZ6R7c z+`o3m@uS_mA6WwK3EL&b#WnT%SiVAwp{6{vmQfwvZXi0}hapxL=%T-atLuF0JR>a~ z1jh~X25i4refIV1!`GT`9G6o)K`TwRNX? z0)N_Gso%IIoV+A_xx{&~N>UcGA^6Fn&gzsz!(jUFz5(}cxCw>dKX-GU5R;@y-^@72 zNQG9TH7ogphUsbn2e4;yO{@Fai*eJt*Y_k}1S7mO=1?7GK64;rlF|-QIRTfJ^iI-qiEyr08qh5O-DsgjjKiL1s%AyjyK?uzWd>5c=pgx-hY{Q2USvIC*vZx z7(IY3H;lT1&Z4`>e_jr_%Ba+Kz$V%1p;PRKR{&vwP1*-tU|M8mXkX`K^X}-+`U2+*UmQFM>Ch@!aBraTWol(~K$Z`i@v{ce@KS^(2_5b%n=^fye zs&!DDmBKSR2WAiLq#*NAA=5)KW zj(GT*5{oO%wbE4di5V1#{t&U0QSYZ|c0UthY;W_lV>v9clGH@WZG0T)>8xV69=;Cd zH#fvV@O*erR_nJWhatW8Sp0g{mR$yRuQY7V4N&| zslbT8A#5*QRPvSt!ogY6gsFEnMMEngI&n|<(_Gmi7HCYZTEpwmUb+g%s@q+ zRS(8b*B*<)UiNbIM$|2^)ZW`wGF#*6@U!Cw`A zU+Nhm6WL}_(^Nk|Jj@m<)<1oAvJyQb<3-gS_OubT@~%SGs6&btHk>T0*mFr=kSj|~ z6VMcEX2lQPqD!?zYvclDdR-f^BQsSfw8@$?pnKgiruq&OVBGj2Y4Aqmcy7^gYi0zv z83*L5P$5C}GHJM@lB%eeA}Ni&TK@WL`}T{Jn1E#WIYqGoT6Y?G`5$)(x~yeT?kPfN zmD!hZ*QaKf1q>rsbCyVOLuacZbe@VZQKF7^K?Mgg|a_o0O(x_c3BD$fv!oZou20Kq1)=53QZP$l

    Dashboard text leaderboard: score per model and benchmark, colour-scaled, with the best score in each column outlined @@ -293,7 +293,7 @@ The `HF_HUB_OFFLINE` value is read when you invoke `oellm-eval` and baked into t ## Publishing Results to the Dashboard `push` sends the `eval_results.json` written by `collect` to the -[ELLIOT dashboard](https://github.com/elliot-project/elliot-eval-dashboard) +[ELLIOT dashboard](http://test.openml.org/elliot-dashboard) over HTTPS from the login node. Ask the dashboard maintainers (ELLIOT WP4) for a personal token, then: From 351578b5b43211a9c010d71501323125c98d4637 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Sun, 27 Sep 2026 16:43:50 +0200 Subject: [PATCH 43/44] [Base] Pin lm-eval 0.4.12 in the text extra, same as the containers --- README.md | 2 +- docs/VENV.md | 2 +- pyproject.toml | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 12e1caa3..4c833e5d 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ The `--local` flag lets you run evaluations directly on your machine without a c ```bash # 1. Add eval dependencies to the project venv -uv pip install lm-eval torch transformers accelerate "datasets<4.0.0" +uv pip install "lm-eval==0.4.12" torch transformers accelerate "datasets<4.0.0" # 2. Run evaluations locally — useful for smoke-testing with a small sample oellm-eval schedule \ diff --git a/docs/VENV.md b/docs/VENV.md index f108fd4b..6091da62 100644 --- a/docs/VENV.md +++ b/docs/VENV.md @@ -106,7 +106,7 @@ The general venv pins `datasets<4.0.0`: lm-eval itself accepts newer versions, b | Component | Install | Reason | |---|---|---| | `lmms-eval` | `uv pip install -e "git+https://…@#egg=lmms-eval"` | Editable: wheel drops template files | -| `text`, `image`, `audio` extras | `uv pip install '.[text,image,audio]'` | lm-eval + transformers pin + audio helpers | +| `text`, `image`, `audio` extras | `uv pip install '.[text,image,audio]'` | lm-eval 0.4.12 (same as the containers) + transformers pin + audio helpers | | `lighteval` | `uv tool install …` (isolated) | Needs `datasets>=4.0.0`; conflicts with lm-eval | | `evalchemy` | `uv pip install '.[evalchemy]'` (own venv) | Forked lm-eval | | `dclm` | `uv pip install '.[dclm]'` (own venv) | Pinned `lm-eval==0.4.9.2` | diff --git a/pyproject.toml b/pyproject.toml index 80afe81c..b6049306 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,9 @@ dev = [ # (datasets <4 vs >=4 conflict) and is installed separately via # ``uv tool install`` — see docs/VENV.md. text = [ - "lm-eval>=0.4.12", + # Same version as the containers: task names and dataset ids in + # task-groups.yaml follow it (0.4.13 renamed and moved some). + "lm-eval==0.4.12", "latex2sympy2_extended", # required by the polymath tasks "torch", "transformers", From 1842d86c60a07f04a3132ed9b643684b213994dc Mon Sep 17 00:00:00 2001 From: islobozhan Date: Tue, 29 Sep 2026 12:27:28 +0200 Subject: [PATCH 44/44] [Base] Fix logged run-folder paths for language-bracket runs --- oellm/push.py | 4 +++- oellm/results.py | 4 +++- oellm/scheduler.py | 7 +++++-- oellm/utils.py | 4 +++- tests/test_collect_results.py | 16 ++++++++++++++++ tests/test_push.py | 9 +++++++++ tests/test_schedule_evals.py | 26 ++++++++++++++++++++++++++ tests/test_utils.py | 25 +++++++++++++++++++++++++ 8 files changed, 90 insertions(+), 5 deletions(-) diff --git a/oellm/push.py b/oellm/push.py index 41d7ea77..7aa005a9 100644 --- a/oellm/push.py +++ b/oellm/push.py @@ -11,6 +11,7 @@ import json import logging import os +import shlex import socket import time import urllib.error @@ -313,6 +314,7 @@ def push_after_collect(envelope: Path) -> bool: ok = False if not ok: logging.warning( - f"results are saved locally; retry with: oellm-eval push {envelope}" + "results are saved locally; retry with: oellm-eval push " + + shlex.quote(str(envelope)) ) return ok diff --git a/oellm/results.py b/oellm/results.py index 3a4ff706..c1e7644b 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -4,6 +4,7 @@ import json import logging +import shlex import subprocess from datetime import UTC, datetime, timedelta, timezone from pathlib import Path @@ -910,7 +911,8 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None missing_df.to_csv(missing_csv, index=False) logging.info(f"Missing jobs saved to: {missing_csv}") logging.info( - f"You can run these with: oellm-eval schedule --eval-csv-path {missing_csv}" + "You can run these with: oellm-eval schedule --eval-csv-path " + + shlex.quote(missing_csv) ) if verbose and len(missing_jobs) > 0: diff --git a/oellm/scheduler.py b/oellm/scheduler.py index 9c6169cf..fe9a54dc 100644 --- a/oellm/scheduler.py +++ b/oellm/scheduler.py @@ -4,6 +4,7 @@ import math import os import re +import shlex import socket import subprocess from datetime import datetime @@ -796,10 +797,12 @@ def _lower_suite_only(s: str) -> str: if local: logging.info( f"To run locally: SLURM_ARRAY_TASK_ID=0 SLURM_ARRAY_JOB_ID=0 " - f"SLURM_JOB_ID=0 bash {sbatch_script_path}" + f"SLURM_JOB_ID=0 bash {shlex.quote(str(sbatch_script_path))}" ) else: - logging.info("To submit the job, run: sbatch " + str(sbatch_script_path)) + logging.info( + "To submit the job, run: sbatch " + shlex.quote(str(sbatch_script_path)) + ) return logging.info(f"Evaluation directory: {evals_dir}") diff --git a/oellm/utils.py b/oellm/utils.py index 663c0269..b94cbf2b 100644 --- a/oellm/utils.py +++ b/oellm/utils.py @@ -97,7 +97,9 @@ def _setup_logging(verbose: bool = False): show_time=True, log_time_format="%H:%M:%S", show_path=False, - markup=True, + # Run-dir paths contain language brackets (sib200-eu[deu_latn]) that + # rich would parse as markup tags and drop. + markup=False, rich_tracebacks=True, ) diff --git a/tests/test_collect_results.py b/tests/test_collect_results.py index 02315efc..a78a68da 100644 --- a/tests/test_collect_results.py +++ b/tests/test_collect_results.py @@ -2,6 +2,7 @@ import json import os +import shlex from pathlib import Path import pandas as pd @@ -552,6 +553,21 @@ def test_corrupt_json_counts_as_missing_in_check(self, tmp_path): assert len(missing) == 1 assert missing.iloc[0]["model_path"] == "/models/pythia-160m" + def test_rerun_command_quotes_a_bracketed_run_dir( + self, tmp_path, caplog, monkeypatch + ): + monkeypatch.setattr("oellm.results._setup_logging", lambda *a, **k: None) + run = tmp_path / "m_sib200-eu[deu_latn|fra_latn]_t" + (run / "results").mkdir(parents=True) + (run / "jobs.csv").write_text( + "model_path,task_path,n_shot,eval_suite\n/models/m,sib200_deu_Latn,0,lm_eval\n" + ) + with caplog.at_level("INFO"): + collect_results(str(run), output_csv=str(run / "out.csv"), check=True) + missing = run / "out_missing.csv" + assert missing.exists() + assert f"--eval-csv-path {shlex.quote(str(missing))}" in caplog.text + # ── --check completion matching: exact/suffix, not substring ───────────────── diff --git a/tests/test_push.py b/tests/test_push.py index 30c64471..a2d5ee23 100644 --- a/tests/test_push.py +++ b/tests/test_push.py @@ -2,6 +2,7 @@ import inspect import json +import shlex import socket import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -357,6 +358,14 @@ def test_a_failed_push_never_fails_collect(self, tmp_path, monkeypatch): run / "eval_results.json" ).exists() + def test_retry_command_quotes_a_bracketed_run_dir(self, tmp_path, caplog): + envelope = tmp_path / "m_sib200-eu[deu_latn|fra_latn]_t" / "eval_results.json" + envelope.parent.mkdir() + envelope.write_text(json.dumps(ENVELOPE)) + with caplog.at_level("WARNING"): + assert push.push_after_collect(envelope) is False # no dashboard address + assert f"oellm-eval push {shlex.quote(str(envelope))}" in caplog.text + def test_without_the_flag_nothing_is_sent(self, dashboard, tmp_path, monkeypatch): from oellm.results import collect_results diff --git a/tests/test_schedule_evals.py b/tests/test_schedule_evals.py index 38a37516..4a9b9f00 100644 --- a/tests/test_schedule_evals.py +++ b/tests/test_schedule_evals.py @@ -1,4 +1,5 @@ import os +import shlex import sys from importlib.resources import files from pathlib import Path @@ -35,6 +36,31 @@ def test_schedule_evals(tmp_path, task_groups): ) +@pytest.mark.parametrize("local,command", [(False, "sbatch"), (True, "bash")]) +def test_dry_run_command_quotes_a_bracketed_run_dir(tmp_path, caplog, local, command): + # Language brackets end up in the run-dir name; the printed command must + # still paste into a shell (unquoted, the | starts a pipe). + with ( + patch("oellm.main._setup_logging"), # keep caplog's handler + patch("oellm.scheduler._setup_logging"), + patch("oellm.scheduler._load_cluster_env"), + patch("oellm.scheduler._num_jobs_in_queue", return_value=0), + patch.dict(os.environ, {"EVAL_OUTPUT_DIR": str(tmp_path)}), + caplog.at_level("INFO"), + ): + schedule_evals( + models="EleutherAI/pythia-70m", + task_groups="sib200-eu[deu_Latn|fra_Latn]", + skip_checks=True, + venv_path=str(Path(sys.prefix)), + dry_run=True, + local=local, + ) + (script,) = tmp_path.glob("*/submit_evals.sbatch") + assert "[deu_latn|fra_latn]" in script.parent.name + assert f"{command} {shlex.quote(str(script))}" in caplog.text + + def test_schedule_evals_slurm_template_var_overrides(tmp_path): """Verify --slurm_template_var JSON overrides appear in the generated sbatch.""" with ( diff --git a/tests/test_utils.py b/tests/test_utils.py index 70481fb8..f82a7378 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -556,3 +556,28 @@ def test_follows_installed_datasets_major(self, monkeypatch, version, expected): monkeypatch.setattr(datasets, "__version__", version) assert _dataset_load_kwargs(True) == expected + + +class TestSetupLogging: + """Log messages are printed as written, not parsed as rich markup.""" + + def test_language_bracket_in_a_path_is_kept(self, monkeypatch): + import io + import logging + + from rich.console import Console + + from oellm.utils import _setup_logging + + console = Console(file=io.StringIO(), width=200) + monkeypatch.setattr("oellm.utils.get_console", lambda: console) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + path = "/out/m_sib200-eu[deu_latn|fra_latn]_2026-09-29/results" + try: + _setup_logging() + logging.info(f"Results will be stored in: {path}") + finally: + root.handlers = handlers + root.setLevel(level) + assert path in console.file.getvalue()

    z*9%>^4V!aI0nd-Ow8IMv^GnX~F^-A3wY6?ZN%-dKe;BG*v{=RF7w6^zvgtbqniac# zee9DqYG_@KI#dL5A6i?%@{=Q%Uq&uG^g8;)qS`Z9j=lIr>k zome3rqQBX+1c|wQT=uKhATc37ePIU%osYO6xLyDf1NqU?VjwdzUZsbGx0D_I(Zo== zBdE*;-YKc8g}dpei^;_q6QnG|kYJ23{-r#*(Eo)W`OB>x9>{UkA&So0Ia0&-;W~%= znFjo_4p2tiw#y^qIFOQps=U7K)v>Sye!kOSm4t5+UA0xEGWDGh;_>K%jzGVLJZ;qv z)@7JnlAwPQ(&Do*LWd=mbpJ+HTh02;{I12Yrx(Ua^?goZDwDitMXbIm{WV5w4B(M= zn_(aDJ;Oe!@kw(n&bTL)mOXQf)jQx+`P|&Vxj~yZm2M61n5L85yk&aDA_TZa)ISBDq0SCD08(NqYWjG;-EpuOh{TIz4zUKHu3_=>jJ zT2g7rtfXks()CHBi%$AX>ompd=cq0qO__ZZMI0OAX~I z(L_R<+nDRvI!s$xj42=?krr#G2}F$v!CPfU2Cxn{>&w1M?1uaGdjpZ{D6K8~ec*zX z%zU@gcMBj>NS>j~=_ErUG=pzNI|BUwrvlKr}gEk%*x z16@pwP{}`gj691D`R3~jzn*ZgTv@#2O2M-^boO#)4_iST4ra!MFk{YCC9L=)KGyVuqTN6W6|F(pd_8sS7o zTYwj`fh1`9WV$N4?64d$j$eAKX{%(*BvKo}(D#;NIBg>&Ch3qWqMwTaLhr`$3`Zjn zZ4P5T4H5}*7rsqoF{5Wb($QQB1g}~v?bnfWue-yv`>l43c>f7ejIMmOBej6oA9{+$ zsvQ<-I-msBcw2;o3E@{VIj}h{tj^MQyS$6^{be$So#n~F46mS<%0k%0j<$KFA)-S@ zk6rp}Zy|f-BNm1oBlV{{VjM#}W(~XOY+X4cm>w?0NrL#sm71z5nSsjZ!x{g?Cv8@w ztOpF-w;k_D{*-D8GFk0JUzn8HON15k61jVOh@G0YbG=xaR$`%Uo`?Pv0^=LMap0m) z@v3d8F}wbYMXOIEIhiDuu;gM=s9UJ7sEj?@@SmV|bE1>Xl-}e%d3uz71Z0^(0BlS|*%iI60{BjP?oaLR zXPwlN(WKlybKt?8|2!i~|Eo?@@+o=Es^Gx1zJIM>8ull6;IigG{-wQMERD(}86^1- zYs2g7Y2yVB#b!+lR+DGTV+;#1W10TFCq=5MbMx zAfB6_?-%JPC&u4xk$>8r85S0Gx&4M$Ca=RtXDgE*QfI1JZ6CgVIhKGaWv|Xp)#^E% z(P+EUA2p?N0!@S%dE?2fG`^uiXH6moC0}BYEp4fb3$x@PYu}!2!hNR5v3f`B5{?kW3f)jfT#Z01|1wtgH<1 zawdFVE04;`E^ba(o6i{x(@#v*^P6e?p}M0jaG8r&ZeA1fZItq_MU=(KV=Lr4r&K8ldSOm5 zn?MAQa$Tc{MWlVJCoR`6x8ZAxWR5R{Q8gXJg9DRUC)S`Ofj*9)YRbQ^AvEwqmo zIqoe7i@gaT-?&J4Z;94x*wIsatOXzxeb@LIzo*~Ouj^>IaU$~CsToVb4z zcmn`0FXz5Z-d!$G4$N|O7GcW)zHbe&WI{%3%Sm}DmAah=0N+9ISS{pry$y@47jy1A zcGrKo3u*!+moF}N2aemZL_%&CO4jPDkB>QAb4^!atJ<_JLxocC>MQ*xbh!lu;&YNa z@0I)LD95w?j-z&Tx<1MAJ3Tgz_uh-`{eC*=`2BvNsTG{Jm3}}63fyeX==$t4vphc< zf{dG4QB@`EzL2!$eHCgRLPH9WTbQqM3R~v5!i$3VEvS`=nRGjDX^)I#ewAP5 zV6tc2v6j|W1!INGGFW~9&ds!$VfxBHu4-Bsk`S8TQ8G1sazV^Rd~<^^Z;vD_g+rqZ z5iEV$p@UMUB!$%E-pszK4GwnhL_4$@E4S8Ww#0U8OTuVmCA7K;*(3v8vDyd?cvhm`n&Z8{ znVS38RjF`Q2MY3xhcOF>HVRo8Vmf(>n`wHD<>lQoX1I-$sj{VrZ6f`YY5kV*d?P{r zJhb@|y5y@V`cULk!Qm{5$-B;P^ud$%hLydB*>@yx&JXDIGuz(-7`u|O;Rlkq{pQ~vstP2Tn+vVi)G`nlK;zL zcgw89^R8V;IC7=Vow5;Oj)9HUez8hrpQX+H5)@+%aE>3@`coMj6&)8PjqO^SdzV<=ql8F${JCfY7uDVm^tsqH)8CZvNBKk?XC0*r4j{^QrfWdnF*2(w{5CVfBnu!?L4~>Tb=bPdCQEENccY&vsHSJ0KC8s zD3#y?F$F*R#v~R8ytv}^?F*OX85UHAKd zJ>C>T$}qg_-gMe=-WQX_FE-kHgfFX*=hLO)rTuDc|bAylTYtCwuyn?rs8i!;Dra)7dHv<2dc;7{(*>}DF6ZF37~mb zecOtH`F}|nbhZr!u@T>yiADYUJvZDNd);?&abq0zzU z%TII!=L(14s592if7BJW2dx?~n4$z5dLlN|q1Om2uyWV0{!hz+f`gSVDDW;Zq`$Z3 zj|j_+fA0?xFD?Cqw=@)ZUO$S(+Tz^p+c@I~Kl~{s7G=&sEj;59u#e&;B4+iiRO#9dVL&%d&!jXV zftNVaz^DU2L5M;g2KXc|STrn^&X*OXEs6U~=OI?0RNY+P2xM!F>Qj(Mb zYS<7MwM#s6Y-r3G@Eo)QW+hvQIUbPVdJX^4<6dcT6zH5AHqxG1Of~ne+Dnreu9z3Yvj@oqsewg zzrl`Uxpum`wG6H}blk57Zm+CNC60(w%_v^<&h-L@4rx_=l5U0?~{sIDAaEALS%T?-_-_Z{5M8=jbJg?R&;?m_1!p-Ap<}r zZijzt%$Nc$$4;BHi{XgWzeY!vv;2GT0lrFlF=Jr_p{fw?a?A(^$e#7#a5#{(@y(2X6()08b=-9#>%X$a5@MfzH_?=@QnFd5Zyt*o<&G2!zs4)6p z*XkW@9UWl+R{XB^MeFf~Jcj?Fzf24nQd>O~t7%F|N+d<@>N6Olm*{5(T=gkNis|C!;XM(j+bSM2l4K0UhJW|Jg0~P6 zo+<-Idj9971g->$;QdVZLt=oHCN7Vk8=GrmlEz0x<>KPXc3N&Jgf*Rj=!_3K_Xc!c zA68x$8tW^Ksef>%uZJy_9FZNM42kZ^yBkleG+NWCgCRxXd-HSDv1*kjf)RU|f4oY| zE3rLJk9i0t9>3a1Q}`Yazw@V2WRt`t3F=ri3?@yan^8#fI<5d!pDZ_c_x^r(c(}IK z_!cjgw*n{qNln(Vl*dk!ikJCj2fA0ke)#G!ur z*zUL{lY2113;kGGDd2YcRC(reHUjM|()cKTC|-|@sebU}A%#Y4)@(Mf{jVda_EcygBX)1AL8M%OhzT?d@v5)^EVwaFnpgR>uvv=Btw}u@b~dXuq{t zX@w10(uL0-0Y9X5rR65=N9v24Rm2kuFeX7KW#7B7?&SYDgz{^2bQyrONjwNCCcLgP!2t&+VtFtXCW!k8ui7LgyPSVT~2 zVW!(26U<@ufs0Qi6EJS4^XcTWy`l`iU5qk%hFn9{G0C~DrhdmgNDnILv9>)h0wB1w zh-F2!{YIxs%LR_{RPX`m$ryezMaKg=1wT$Y>CN#v_T_!S4LSc|Sn8y=n{9O&zsD7r zgWvVEY{OxXFGUmPa=5H#BlPWqk7$qzYrZC0 z)$=B`kU5C}2Tz#}?oUIP=`mV=tOk+glX(Xz^I8c$@V?s;Q6gsG42|CJD~vRWmNqtuQl9YZGxfCPP9o{upwV2teuS0qNK`vpt1MK^+$<#bj+p2u zRe}Mo)uvE=lMCru{LFCkK2WJ^Pd6pYH%aU3a;o89{!e~(s>s_4+GI8Ir4;qEK@*Y0 z6flI!4V^KjXFgO^C4e&<^m`kr#DIUKgYHfHsS~kTxTC_!_!0Qt<+nZmnj)#ly02K} zaVI)bUA$Qs>2(9Tmr}M&eTkT~RgTe^Or>8Uc7G-rbPTQ|E2kgq7kfbr2Dr)J7o|WEM|P zbaNUpG4D>tF>j3smOXG_+fCYXJSuk^Hy1giXH<#rp&zk(dkWm$S$+PQs+z@>u7bS<>nJ+lik`X0AE?oD$+BuW;~ygVm%9ss{sNZonjV?`me<*Z5E&N zo#EMj)FE&Cu58X2O}Xl-D*n@8V!c*ZojNN|AT9byE?v9SO?T=&zK{i=%-NB%Ih@m` z=r=h*wnET^6}WRMh!OkgJXu?|S|Mn{>#eq<|9bj9Dezg{o|4;l!oK0`0;Gn4@BJZ% z`NV&GG>bu_ruY}1om3cZjAs40NgU;QrWX}~q8p@lBNdG6eSO;IIx5L<|9oMPJc{0? zR3&o%a{`DpD)Jsi0g2XXow{0JCNLI~I}Q}|I6D!1VHg=G8lav@Q*dlTmuw zA8mTkYjZpChs~aqZ(O+YVFGNpko&@JP*^lp20i=(X1YuN)yPOB5Umk3x)`J!Cs3oa zss(&Z=wdY!GeZ`q>jMDiWczK2R!wBlO3RE_nwv@6iA&y`uHOS8BM?nh*RFjYBU-y9 zKs5%1MS0wwXq1(echg&rNLH$>=?_nr6t4yd#ugp<&(}ioC8N8JM>6(&1*Hmj-JW)0 zvf4^M&FjDA%&*~5n%qpIqdfZCb_S%I+D#tSr*Rv5*AEYO(Zq8PmHwzbmGu0G8seogvQZeTZU5?kpFL>|83au(MWKB)wL;9D?)!Epq zL!N^fMti_6&ScJ5xTWMZ9hu9&J=?fFPQW}&>zg>y>1R2{*I29d`P++lDO^%oibH<0 zVEPSch+7xaL1X_QDK}z~>9kixIw>n)pW0JEn(`e87DqbABo{&TyWOl=ygwOQ*=l zXn(tBJz1J&b;W^>^;*<45o>3#nBIUCk%j+;`6nzOh`5Af+?6{&MI`g`CR}NIlK-|= z*cUlc;%24%y*w$B>*ceCRab17!QHmLlJrV3X2ey)sBIo(dH!h~(s6>LR9(9(JR7GUsikE2L!acv-L5@rsz_6X?LGTo&ZimX zvlR?&Eb(gH2IJv5;I!2RQb9tV%})2driwr3NA~PXjyIaTkMMagi3DvXysJ-ETPp20 zegR3Y7a+90SdrXS>IdC;XtSaRM)l77c6p)@!>b9>N$Bk{map^qIb-$na&6%yfsddRdLTZK!3PXvi4L>v zWqsdAMS%y?TrscfQ{-{fydlh&O`g0vgVyUa!imdsEt@crhi>x4wTtMCu7@ndoOafM z9w2@AE`{h-H5J`h;0RbzXy{C9X6gt97U~4iXrFvyv0Y4~GOq<%zcnx-ng4km<)5Nv9V;DT>4o3;rt56Qp7R?uqS5|+J&14r_Cxc5v- z{;?5FHIj7~fYT{eoT|!DDK0Q>9?bK_YbYM74{f^frtuG8;UEFs_4%qp&cJXt>-bY? zTr#6oN>#T5B}}smhuXPsmz+_<+{h9z3wz%ooqYMNk(>5*ab)dnR%BhZwCxl7Xl*^} z?_)sgUz{A;|7qXXxIBpk638)wb%J~if-n9qkH%f1`6wdlC}g5GFl}Z`1_r&EG$>K? ztf}~j0)Mnl^*+IFl4_?-mG*7+1fM3Kq#TwR5Ss0H`2@SQQoL@s_;*IMmar~i#G`FgG9SH~VGWp+|QE1vE z3M3%LQibg6x^I4L;Dn~;@HuD;gz;#4ns{~y%vu0NE1eD>h-I%VNrYOd7?JgBYJ_L8 z2SyyF$I@nuSSBnb(1G#|(nKE!7AobKUt43As;vYkU@R2Dz&u@*{eP?#^`*Sn{Js%>%Ciu?ccX(R)?5O8!?Yr(} z7VU(P-UAN>-jy_zYs}B7LY;NA_eHN{;X&mXYFWaUPx-MKA0oT#t|Nh{b`i@9->&Hk zoBNH<$1vm9y|Fs;h%8rd?8QU|YuDuhqcrij414;p2doSqq$sf64)7}We`~R2N!Y}< zk#N=0Yq)#mr~QVXi?du`!m&+0DF`dqKLV4afhX+nJ#_KsUWIF8ubUke)oOL~tEX)qm0aXN9 z$o$IM`wu>h9F^xlE)Zb*!Mn zU`p2uW{RHYC14{5MMgdwgJf-f*lKri_YNhXdwVY)eWc!{L8c$00 zYw2iEbAN^uDX5shkCXl(=Jiz8b3Omn=I{8RMJS&xcml-KD$Dqr-A(|})#J;Hw5`@i z)4GR3MGGbcx7E@R*jk+#*E8^k;u4Z%uR|ke*}~r{nZZPaye4yl zlBfseQrQl_gU=a~w}B9BPPJa0?&I$r=$bkUut5ZAr&-#VyT9?r@G5~BX8c-U%Ho0z zNKrnmja{;k@>%s_!w^4%9iE27+{4MS6OlDLl#dz7D0kHxO3(R>#mt^tlitH(ZD}Tf zIU6X{^>bd8w#arF`CDka%+1ZslzH0&pkqb;VcWmF{Ty%ZHNt)Iz``uTwf1MHQk1Rm z8-^YiU~daOpS8I|y2ysZV^W5XtM0d~RjsGiK*t8bqnPpjUZDlW$(nn>5^Z2|h#F(c zf72R$v)@Oz2dI^GMy=i)N{B09fOzjKPPY6TxnynSK9XwS0{*C;N3chUXwfSb+Ctw) z3@olyhPKaq3Amy*9w6rKVU-wywcOT+AA$2+Bi+0H%qfhfk5yCD{oo@bRQ zwFUIt_(16f3%B8ne~d6*KgM55zt=jBiOxHqu~FcqfP;Fn&J6;Z6i(Nbkm&^+uQmCr z@y-0OSdu2MUFGS6uCC`WASyPGYmtdBo7kZ!xj(!Vp z=?$>#h4=1e6@{FWzq>pyC*7{Ot|Tq4X)81BN8};8pDrz+{i!wEz|cU+(HqL`{94r! zxKQHM@)e^%74|bzw2)eMK&8Uyz~k3|b;Wm`#AFh)_&b=#+*tivn6oOKM|)k34BC}u zEZAS6fbb!)j8pRk-sB!ZZmgcqao*ZG^nBIk0|$xk#acSwN69o^`6n&gnonwdqJ`1_ z(*hXb>hQyxBrX)t$_dNM;??^s8^)F`*`42h&Z&=o`^aXjx`jPviZ`i(V&OAZ51zz{ zNpsWam^9q53PLXSV2J$eEg$JeuseMV-wCoX)LuW_OHvm;+#3p7$#9{#kml)^8dTJLn+(XO1O~dgsE?O9b76P%2lv&0jOojy6N{r%8qlTmcS&y#YWb>33Q&CeDmeo> z;&9Nd@Kf|)2`r}O%IstH;UA5d3BLP$?(ShCNY+6~+Hqe&4r`}jlZ+kVn zVYNo*$)b+S?{ciF<=E|&jj)L&;C7^})MDT>p_4k}!GW}<(VcT+B;>%h(Bixp6oxTE zxm@^7|5(NSc&Vka=+lFljREKG$ufRQV&pSET(|W>W~Q=I1D>xEH7=t;!|)k6q2Q0f zt7wzYuDyZhVJTnTMr3G|f?gSh8Yd@jL#~rP;8@5@OYio`@F_dxEGJVAEG2G1{U0au zdS4wDtEp7j8cvdBpPEbt71ZX{9N`@k$Z4)eSKP7kRb187sH~?mMn;w!J~QK^Nt>s- zYu8x-VVRe4kD(8a1&9-|4b^F!(L+Fh!18ewl>J8A679Vj$t~-z zVoCuqo6V=A#xub3Niv_hcFp{z^@gY|>4VeE4-Es_F4m!{$AgqJRfX@6U^a&T^hGP{ zGKe2TrP$pv&x^#|PPPBb^QtMVzE-`JSojz-Th2Z5LBY4-VTQ*2J9vfD&Stf!)hWc} z7|2}ydVXxzdAmiO#qW+k|FJQzBMsEq!S-N0-0q*P7e#J-fDJl4qB9ac%azY)JOPIA z8hh6LPkkL;x32>m&x=480PlXs;vcmj9zL?*Tik8x{Z%jeX63>3<5IUo{eqMcjVgKBV7Lk~;SIO^ zv);xK%xZ2p(v>U9@~%C^S#zfXI2-)%X)_AU9Fk!lDXOp6?0iE*V<0h01>3x%?5kSK zRGtdgbtCn}pgvhE1=m(u*WWc^Hk_WZ{vy;_X%sdoIMQhg(6wJA@`nSz1!mJ`ovapJiS^Tq3sZXu zi4kFdNPUVPH!W9q&ok;ijM--CxfEjI(>bwuM>!g+mD*zcHh^I=cFn~Rzm|#8(DS_Y z?}s%+&irp8A%cY8pke5hdmHXB_WriL07tS&DiyzTd&UxEm`@nFUmvK5QZ| znUGZf9?PCz1=Fo2LGKu-;ge)4wvCH(Fl69#y`zmZ!N6vrnEC4pea8Fu4aSQ!#F7nU zwm)rn9SrS(An=ps`ue7Yu;*xaQK< z@L~`mG>ywkmJC#Mn4t_0ScA>2t!1^w{(-xhZ;XUM|72DoA|u>YO0va3nZMClPi z$xB^Q`iv1`FATPQ{tznFrD~uu&c|D!j$c$!_RS7lUnI!{BCkzNV;eWhuSMTY%he|{ zB>C<({I|E$w=Lg5S=1S+mwtq?m$H}kZxDdjHJD#t0A|Vr$0mBEUv~^cH?Hw}F@#G1 z6i;B%u~w%FB?>iB|FN3LHP|Ml2GH7J%0 z)O@Gvc^hFp0FN^oP+yBFP&wvlw)>(D&M?*%lxgywTn8p@zwPV&)t|KuU*LTQ=w51O z!48j3wpq!v%oNvIM`B*9(%Q3D22&@aaLsE)UWNeeveD%3{_|SPNBr0`7yzbYc{r`{ z^()o}5B=EPRH#~;wc3EzE^F_6i^&7`IgM)Fx0U4W!<#>F`4mru$J}|$b`IVF*E9du zEDfF7jJND+;z6On@4;{ycJM$$m4+IOr$t{`gh5nD=Tml(C?RW!tZv^|8A*1cs#UVt z28PzYv5*Rkn;vs4kuzQsp%S>M{?dJfFZ^znNd}qh5g>rypKCQ~ZFe0OO{#!{J2BW&8S~^9vo2}N>>1;-;4H^dbdpc)( zDcLJQ!qHDajaR+kR9ryret6#)WGvS9`nV^v#40q=j_A=Gc)?Y>W`{BkpV4fBK$8W|(jHvRZNZg*jS@pxQ_Gi}N?wnVpbV zhsfhKoIjOmkf74Sm=U^IcVp-W0!H2*(@;3AWMS_syf$%J+;3PIOL8?WWfpe&8I{+x zqzUS9HkFc2&uo^CQ`Y}QayO6EWK&qP2Ro)cbck2ey6yuGHOFiXwnKfG(cHl8%o%mQ z@@Fi;$)SL^0ti@AuU4akjOgUT4^N<~lliud7;>F;|9yOVp!FmV*_&6%8e#Qr|G-xB zOq9hsHed*}{D7r3wV;IfU4+PHfLxN9%}0H}g!%PsHcTAMW>ebBZb{4>oETprXfkib zWGE-a62tCMQQptFAV7d1Kv`3vc4)`3Lg{4A7~q;dO@tam@v9|{JjF26-!UY3qunlT@%HJq${C-R03 zsZWh`7U^>7&k`yg5>^V8X(8R4Q)7tEmYS*?oX#mYfO&qnKj_zBk!pZ)9eN?~etaEx z(Mm*&=_vfBJ>aoYI?K_z2wflD9Fw#VO@EXV9)H4Uo@}tz(CB*;uK0lA_*%EaQJc zY)AQ#Psf=Lqi3gyr$b9<{BZ(~$z)Pf+fM8QN%!GVYe{Qlpm7Vqk7u~INq>~`V zyn0-*5o^wIa+OAbO(dUhZVaL*c^c&{OP6Fy%gTW39(15tGE6z3OW12*S5hs6z>ESI7o{2o(EgTb9T{vH@=R<>Kalwk zI2Q_9MP*X-m?f2m0m#1z|2ZZUO(JLC^nu>StsQfjK*y>Fgo>AFg^|!-+P_mX_@@D9 z9k(IGixQUK%;7S@vT#^_^SWA1%?}zhKBV z3+!Qf3r`8>kx*i4U_0`hFOt`pASGmeYVNFx3E4MqrNBydwimIuih}F~xhH?SBrruW z3dDCQ#nn|;@ANtB1>6Q=?Tr-1wXK0VGaFwu+X7DqXj|0#hUT$|A9g3puBn%&<-m2k z1k$nzC)IM%)I1D-q=^v3P`x(N&k+FaL6Njg&-dj3J%1Y_UT$PIXfpphLB8@2jYxbC zh+U089&`Xz#`L>5${+B037K|sSEI-{O~SZPtl!WLihfZq(yr*V+5NnD9hEnA6lkR_ zi|IY^g^`r5r)1!~Itpyp$>K*x* z%xecV5H)g1I(UdzqJ_HlP|+7;MpR;_A7naYpsDg4o9YX3^2Ysz3&dIN#nBIEX- z>H9x2GOtqicEFYMMuHn-t*6GW(s~$}L-dlDVPS=-eT>qMKPW=AvisAV)9@ll2D2lN zr1#kVIxtDc35@)mneodB^&w>7{L_hfBZyZw#KZXRflTU>v@zJcdEJF$?2ojcYU{u8 zAh&n-J0H5`GLa7(*lxxyX4s8co7m7E-<_dRh^{_8cxanIiCu}*aKbvw8d+}X(ve0#tP{BeMkG(B&l%s#RzO)7(7 z`nbuz%g=rN9!jy+%B)p;)rtJVdmDsVUS3Y8K<0m{1pxHtKCr6SyVF|(53pxgFOg4C zE`I!Ll#&1Lkn{dhTSz?SO1#)V+dC z>JatFPk7kc7OKKz9`p|cGzU?h$w`zLoML_-CgUGWeJS(l{sH%H7htN=IVlWXO+h zSbjYCyac~u*@n`r6V{@p>{uXubH`D*8=F&_L9hPwH#LY&$vy5=dCXM1kZr(2bG}BG z0Yxn-|71KyBRo85g%9g{^sabJK#mS4Venl`NiKNCKHmbMF;WZ zeeTP7?dr!Wng22F?N(V)73hHTG(i=W^&7^c8t;)Rq|FiTFGTzD9-lz#Uhs+d64gZH zaTpU~H_~{r(sg~}?{RX?gFzvHG^8^)n-7blv?#~z<9RXU5 z>0TM)iLi41_v}dv`tsmZ9mZ_dOK{!r7>C$r7Oacq-YHDn!oll(S zOG}#Jj`;G4sDc{`dT}B$n&`yPVKKrtYfD~uTq=u_D#mFNTYWA0+Q66EXPKw_(B^ai#JcyV_4(G$VZ9@M=&4+ z8F&%?<|VGMhr9LX=&E+kjEOoG9?Sy&A?ugiwyqrf>0II_&L+OI2P6P~E2$XD|D?p{ za%a++&bvnO(s`pWi$O$&IT{nlX)>zQUe}twNc1hPf&p zwop`7*6gcdUh5($$L=xrSso1*R)9#)9L$Gcz7a6J_VwTc1tvbpbdtBk@h## z!CNtE@6A`=>>s@2sZ*HXtKp0#bbI)4T=zQ?UXS*P?5(2k^FFcDX>xk8Am$u{tp5c4 zO+y6)p@t|=QilrNJ(b{@5!j@gMLl`>D7e`F-Q^`Ks~DbPE^USnCxoFB({y*eJo=vq zICOhmKfYe}MI;Q~LT7I?ohP(CS6JIzp$|Jb6a)1mG8-?qJWxEoxU^g0X>7e9@PJU`EvjoLll+xtD` zzXCM$bLjgFmX57ugV(*{kCl`!2gR|%F9k6^Tm6piYp{&g)`z;uu9B+87pTGGMF?T` z>rlS`!*s0u?AblE+J9d9)IWa5>-Ok=i5e3q!_09S;A`NTuKp4E@B*6dKYg_H3%t35z!l-O{rWEW%adqP~2acQ@U(AS}| z9z@;o9x^$uR@>bJNQe z>@@@4Ch!`y@{&TC%BBx7)a;WPRnM5_2)`+9N_QV8=J>CB^MHRAiLtU+d$A*Q4w%k~ z;&0r!18M71J}?zrzhlJ1K<-Co)1QOVVcW+snW{s*Q@cMjOHyBUt0}B5NX_SJXWna* zC7?KIo;Py+Q*RXhXE@hs*z`rp`+M0}_0i?ZF`pA_Z4P$ylv1 z9q^K5Hxqg=fFQX&{n_3j`|)%qALTh{0VhkKId17cls3JzFq8SoFNRqLgj=54_iZ7| zQ>!&a6Y1qMiqazR#0nj49eTcYguIgQx>lLM<@^IBB#JFH1|@lSz^PgyeOx`&aB|gB z8~89LCYOqbr1|tdaWCz5>(Iyue-o$pF2QKj%Xp+ib*y7tZhjiTtTs(%;=n9h3fIT- zRiL&JkxT5I_Kx#!#4a<#>cT&Q9!%d^H;YI$aKIzGKhF@1M&lT_{H)tQj@tU(i$u$42ev|FrR@axOkhhsW7w_(Di!*)lY9_m`oGvE29+p$=n@^8`{8H!L79*z6 z)4>h7K&-%CYR=0Y@~+wLoyuC9>+8$yYkqF0(L^RD0du6XKl_U@I6?7a+$8TkjgiG< zwu|GXeW|pVW0?{6hsEruq{)ZHQz1AJr#S zcu$*eh@k@v!Ea2;rQ;disIsFvqsfjk+d#EByAaEm(u4xOjSVmb9}M5Yx>( zLVIYnE}BKv4yd7*Z5v=0eI^x@kfvaYT#nwT%P_*SfZ73qS9THqQ``7 zcx-=JQgX+Hc-2m{@8DW+?YFoU@%bzLwi7ehn7P`}Rg22`8P=s+cGuRp?5F|y@A(hg znbzjqno*!Zi;~JQZU{#aOWs~9b=^m2X(1+C9<5pv7bpugYZ0|S{nwmQ;rG+p(51C; zX}YdX$48LZJ6>d%xm=7@vj3-kFap3neMv`0>V3R%I+K>~8m&vWaH(MSAuj(xM%$&z z$RI+-uK43(jjXvM73OaPpoj{(T(kt4jL z65Yz8)!7q%0e!GJo8}d3`T6G%T5!Db(ZtZexox#yZaPw(A5K%8t;0=6uUFA19*{NA zCSMB<`ShoO(Kw$Vve`m~sZwT*A)Lb4;6$W$uLP7zSxH2t_IZSmJ~&VS(vIzG*JG^6 zg3(zUWH7cKq7oY#3e%GpAJMq1w&@zSWPFExUdHul{eBTIC(WnmWVN}xp<1UF)8A_G zh`aADW}U7tmQC81qI`l+{a2xNJ%#yjf~|vt-|g=20ntR^r-N*#E|rs+m8VsKXG&Q|U3I?tztJeSsAPS^eKpWkVl?zhm?cQngY4woB4 z1OCN%1@1yUDDSMWKHay!-{r5rk4^_1((zhPMPfT`fz2MU?m72N}x=V>LJ+!3FnZtUdNWXMnbapPjvk7Np{wpg|)!LuO-IL zoJ#?Qq}wnq>tld~b`2c?Wd73!uzef44G)@(GEmFJ+|03~te}D*ctH z0_ziV1Nv+K2@>ESK-@QJO7slJWbXw%=conbmk%5_$JtY!{=u7mTuw2_19+?ED6F}-@e z(9muLc|vJBo*^!sP7;OaLx1ENqmi?y6M?ve`WQ;?sLXHTV{v-ZXaFc37GsDjAPnZ5 zy|4u^xx!7J>m8#6sMHY}-b6m~=fwThceeuL!QT$h%XO2ItTDmh&+#VKv?Gf{YUX2E z4|#>q{mi9Vnzkv&edela$DO9)Sjw0}$NL_7nNV}X4j;O<2#CuJSN4?piOC@}Na!kr zbU6*VQ9^Z~WB~1()HUXfyPz@GhqKsBkU8+>p%lwtn2%Yga9o?IaLhlpbCj6u^_S>M z6c{GqAFCi7jBXB1L|%{Z=aGm0va0zzuKiEv*h`;%W-BQK*ud;A8UFm%Mg2#%@K<39 zBb4k7TE-ydiCAkeHNVk-#?bt2i4N%7+5Np%QC&?bXr)wd99 z`<}z|DrcJSv-c<6F14(#?X3ORFVZ6A*l51Jzkh#FzTQ9=!QgzW)d&n|BMPlz!kZ7~ zDO;H2oL(~p!T9*;=Tmv7lWQ%9nz=hiwL5!5q)eI#N$)uU3(02B``>>35bm2#P@qtW zSr@twP3)?)aE@9jjwC#caDDrmIk#cEQ>rqk~8&)^4J)q+Z6J`v@AH?45DTISCBa3N2_-bqq`PYT9E z*CRr3V$l~A%P>8;qsk3jp@r3swc3CzVq1AY8jUjC{)LPIO`(a!VIF)6*&b20CfAdj zM2vm~PmZ50H?bhM->3}udSmc04ag$YG#-e|TtoS5dX(7l0n~Kt-%ut@VN*fdJi6Jl zPb2SMau%RN=VasCR=y8&E4G#@2B6sy=y7|AwgX|&zJhp9W7 zpTdi0d^6>5lhK4R!fD1Q)K@OvX4Ao3YDkK+w~1Fh{nNSKTK&YQfMyRU#SVJ(XeG1P zQ%i*-VZN0^tw2RVY=K3yN+;;i?_PwcEAHVe(A9?iPCNVotWs zI52Xj45+ft6FAr}V8%AK*?szLQuUdO4k!|M*L&bNNYk?3X*mT?H0XBv&;Ws9z&B=pgwxq1Xx$K*}L4EePo`s6LVCK@6s=UMsxBipe$` z?E?6bQF6h@9i&89Wq$@65pHB+aE*xV@2~)dHC^!*oU~ahxweQ*k{lSYzAZAN=DE%f zWpkxV6UOf478J%|oj-*K}-cer3~i zA32*57bSZKM=bgD8gsLWnPr!aO(IrWkUB%96kXm3uKe;A+Y0dVvKsB6S$wAi>a`ID z*sUzs4vnzLIy~{V-tQ4Hm_l#BUxm;B^&yvw4RO=wn?rIR$^5lhMQ-j^GfL?oXzJFz z)vv&@xKyhg_buAQSCoa{f!z;V)uYeD@#L&0{6K4hLI-GOgZ6SVtxVKp3RHu$v-!j0 z*xuf*b|=sGa@W;-IM44Yn(d?UJHG7OUwdb|%23S~=wB;#V$|HeXt{m5y~Usmc!^#X z&*Te`*yZjBhou+lc1VtdElF8Ux_@6UK*E(X`~M+d&;vSaQMfF&AG9w3!+3S1GAE*X5YhQC83H-;2Iz zdG-9#zIKxFpg->Hr53919;pk_vQ0Ln3RtpAfJ%SGfx zGHw3)VN#PQ4f1=q;*n%%uAS(PB*At)gtd_WUau#^&H=hIV);W|fG@@*SvDx4b8h`z z=b6l`w^t9&{&=dOb|Ljos?k&_5$m`;_RN(XvP~8#p$;AV&beHK))Iq7R@1#oqHCq?_td8tWtueXa*Bk+l zvBIh<-gO^(Z;zRtU~FT*j;pcpoArsz^4Lf--cq)|9kyHXPHs@2P<^OdZsV|$^L!fy z#Ms(fzI%$5-DU%jzEb)$tx(r=wUP2^9O}u+ZuY?5=sDM}nt#6^Ky}q+b2(RBrozg^ zltmV}{yV4VHq8p2;z2M@z1@v7Jc-{Ab<&hkk>7gbjs+oTO1lJ=i)b$2{=L?UPXt|0 zP=g!=xq2U6n0MR@eUvOL7yK4&$YD7Hij#=1bGoQ?;t`g-HtlSS zzn(kx%%Z1W)jM*-TV6TMG0aoj&RsJ7YKhh>YlZcO6#=-mB5!W_<@_9b#=b$G!k zG_;Z;lEhq>&#e{QXh1i3GIS=WQgWIU=3^!tHR*7DrlfmQb-~Sz^l1Pw%Y4(l&`1Ia z_Hh-JruNx{?sS^`i$y}v7E-^4)8FW09?54JFVWGVGKpFcih2-}hTj;xX#@oje%2{7 zJtMcSe0;Ny>gZ3Ze5u11?t})d6EQL*&=;taQqa%jLY0CC7Vy@;L!~Zav(`qpGr&kQ z2#hhL{-Z)+K}h$B=IoDh@=xTrZ_OM;=SX#kOG~-R_ROzvk^!SbV{9BX(-??I_ObVw z8jXK)w*UkGiDCvu=-~T-IoRuA$I7Yok(uuc%RtCrHmibb%09cE3v*<%hWt>$Wj@sl z!NT;s=7oKb+nGr_4^yNJa7NkUQzVQ9@*Z(q-FG+5(g(_q4H4`B#jhf}S z34;1323;>#_E3H4X!Y@u^TD6y;rsVzq-mk&5K_{&5H{!KPUrV$u&B`GgLyfn$IjSd z^>6}tv&X-s$7{l;wQ+SfQ6*KUsTC}GT%W^%^a8?0*E(lznbH?2zNeuqwM_6hMh7 zSJM`W8h)7^+*u`de20j%gi5a%cIrHr1{H|JIpu_EVt@;%g%8>yDt&dPcX&Q=y#0^mc`>rR1O2(_&xms#d#RRNlaH4 zG4pba^(>!9`(}s}#wh`P#4h{z1)gnWZDDCYCXgf5s06`20+zr5Q&>1H4t`K_AN(I0 zfE?QoC6|{J6+K@MF?)E3T1M+@++SbGT4#~Cs#KGHnYk^ccD`0Xj5$0Iurij^?2q=} zwyr3&_U1+o!({t^GhVki@EDc&F{>^KtJQYe<71F*zLcikqLP3xqyqBtUSXM8xe{C= zwy^BoWcx`G5bSJqO}QvcxnX?}bQlTjFwKw8McQA7^QY#5?VRO!;2xTW0FI`Y3ou3gYk+YG$cS_d5*iEbYwBQ-In+R3 z+n-~qm*()w%MK{G`+<%~NL!_jE#{6JR3`mD-((;+f5{V--UbP8S+d9 zJFU4rdMWg2zeB-pQKLd<-sSt^Nrtj%_$I7m9n(OakW+=rM$o1*U#sAGy4Ib+i~L`whtGkuw55d%ulBN{;$>b-9{Q3Zd{h z?QfsI(49bLi?jS-IKv2yqRa$$cc4un(W}6CEL(Thl~v{nBw1NGekx!7?6m_No-1U# z)l!^*xBI2ZqT^wIrDN8)noQX7@6IT$eU<&*v@sxGePe3fI`zmpHBn*&XTm|sw8t(5 zrae7BO(_5!GlK*Ha7r9xJ_|-%>DBMXafqc00ih;<>11U?I|3v;AX;*^5L*^*0h5}k z68J`yjAWl}Or2|ZZF=P5OPOOYP4guP$w0{;E}mTNs%b7#2%O}9j#fp9K7twgLmDD40ytbtqOkm|IZPE4o4|L z5i*IK@Bpr#a7NJuOfh;~5hRngDmmX##h)Ezzd#_Rbn;s*s{4AEv=g-tjZ9SAQAU;M zwr;g)X?C2b3W>ErRg-;Ii})>O(TaP-hiMd{7cU_doqU@gt!mJBV}KBP3Q&-YktG~$ zYcj(3ipIM6OK=)%vtpWwBsp!pL$wyokOJ@6qCsfo(M(L#fwewfQTHy3q-pbdPsg8N z7ko@8$EFtKTXr&m7RL#XRQwpOlir&F$KELtbRz`dU+o(?OVg=hVofrn(N{!>NK4(R zwd0{Rm`osPrgvETjKuIIaj}xG#DeaNBMhT(szBB|tR!O*7D%U2(T6mYocy!qj>)OK z9+9nYr&-A~40mi}i z-ivL#)_6~Ay5;F;bXwJ9-TZF&hx)xVhrr!ng`GN{-j3qP`b!x``a$mBTDKwX7@;SO zbT8FXxwp^4x*Zzzm_|z(B=D7BIVr;GSNd8NWVCslJd! zZ}IQU_;l`PBSi37tBH$|vK$N09i~Lw@e`_kXjW!JF>ED`)DklyB)&GY%27Ptz=@-$ z*R4SZf-|0*4l8_~LbSVnCdjy$6Z&Z4f?6GVk`9)h@owuF^A-S7Pl(B|$okK3673W3 zsmCZ5Jf{uP^fAPWniv?^ahmr#Y(gG8LME}hn712XZ2}zb&8uWxgEZ-|bw=6%zn?OD z9O$qWr zXE77JSasbHY@7Q+s=51&h6fVWBEoE#Jkw6?ofqc+95!Xr56nvn0w&#dTe8c%-5Y8A zEEfJglKO)*oZnjhV_?Y1YCS|fKj-T)7BURm))ElCcRyKnxB_F~qV)+2S=g5U9<LeMDq24Z+5)kw`ET($1s!gmN?z?p(x7wB^9>|0zPzn z>33odihX6ZH%xfcO=ztF9CP~Z67JX8Z>uNbk}}MSDCGF1`l(*Us7vGwNcIn3L-)MC z@2{`#Pxs)dOvbmR`t|QnPl#yT;csbD9X@a2-vT|J&ubD{3>$2&GgxI+MHM!N-*52p zC;~YH-`-zu_SYZpc%7dQv7k!-?ts^bWZFn9N|Bnjwll(CUF4i$f3yDbu6Z55Tu&b^ zwT?r9g(c@@$km?e-sJnQ{MQ%3^{Y?!%&CT&n%hW}{_ane(gRAP*W*i(+;hlx4Ml}PD$L;Aj-AjO%c>{y_qVO{fJeT}dr`!kq0g=7rU zYzr837cS$xQXSg-P>xQ^q*}JdoM|GmUpcL}!n6R zx*@$fPNqYD`4+l+XRxbRatvvNCXR-i8a_*kI%cR|Jm%lH1NVBTRPzq4%zS@(Ur)Mn z?$V!ADD3euI8&VO--g))GVI4h_64l}W63H0ZE+5&#s62HV4&u!5W>)&T#JytKGlo; zO&wy}$U0iuT5=rsLb%_!AM>7KtN=Cdf~G!)ImH+|!*=I?@Q*lNu6ZyH*lpc1`+6PE zi%HDM@Q90rMONW;l(`=O?tIrDMPvK{7^g;k)B{k z&RM>D*sTxjw3+=dLNM?-VOb4lqvM;=)Z?D)8xg&yPr{DKgP2oAm#FN!4r)H|9s{bw@pltYXvxhmxrt}dp=)}P9Fr)}IHr(C9!v(#g*!_J#X>Oni zqX_oXOHlP$khc?z>hLiu;YRw1f{ZBRni3c;ZkD76#5A`D(3}Nv#1Z$}arUNtMxr{3 z&&+gex~k68(;nCKiD8QUC9AfF+s2F)n>k^4$JuYR$GJ4Dl<-xv3q1@E@yL%(RVy$0 zSu7r!32z>wn>5D!MXk;kY41aE6?t>iGXxI8lHt}!3~Lk-)Q`QBK~xZP*+etz2*Izz z((=CLgfXOZi9{)@MJeQCFE6w?t|m}@3Ms$UE)78(+A@2WC^ z)+ZY_ezQlg00|NbvR5oSKUuv}1i9V6m-MO&UPyGT0(5S1heTBfQ6ycLWJqZwvx!7L z%qrnL^v!r#8FY4Bo~;Zduki}A0t78w|AU)B&Ca1>KD4k_ZMy*m+*%_@s=wNV5+4;v zp;12eP>bDsaxGduwy;Gl)P?llHA3a^!V~$jo`+yA0k*WoFk8n!t0f}PD(|D+Ox`xt zZ3S#dj)^i(U86i&Pf2(>J-*9VQ*4ZmP|HB1^bFwddExM5Vw%Ax);Mfqv%2eoy$!>1 zCwD%EiH_O*zxzR_;1HrmO43wrn2sv32B540?*F%Qoo-!BGp8Gyr$+9 zN>o^?T~{;EL}Nv^w6yg1yj%u}|@qqcF00O;pA7RRT={I_g(d*_Vi@ z0f{42DY^c+`IhS%h>d+n#CS9~4NTVGvRV>fpPFQe-J^F#Tk43*A-EWb+*le>6}cs@ zXbi)USJ9toGRF1yz9l4@Z^jN43_hj=g@#oH03^z<%^@b{?GdzC1%zyxW9nI)aL^V zRE{~*&`=#KmBt8u8&=2`I%`Tj#*!WL;Cb06R|V}>8U2r|{ka4>I#ITb<6^O1KA*aY zuW$SwA`H_l8-2?47tV(@6n>NCC*?zVG&;?G(#-pX_`Ze=&fbfG$G1AX(4Zzt-w&;H z8Q8#$$Yv9e%t?Fnu*%?>A=~R{`MVJCt354PIcpQST`te@sEJ0EnMdoX<4M7>BI*TS zx|0#=NO1mfO%tckQC+^J_QPon9^w7tVJ_t6`Tg{KP)EVsys2<*W3-*GDM9 z%1YFO)Sq8H6Vf#USwwQclg!$5`^AAZ535J=b7R$pnMX|B8doaS2%w3ZSpseG2%CyA zJEs?MnbI1z(8MpbmYPds+a^GaD-50U@fnOP%B%C+NmX93JOBC|>MK{G2-`c8_Xs`+#JW<>o5p{fI;0})&D)6v=0 z)8;M)qTtdQBv4?Qh=@WGq|CWE0nn8hd?_TP}|$2j{#Mp7%ha>tI@ znkJ-R7KHCh{{&MgUXnoBO@HGdKt!EE9` z)aEOafl=6FiL4Twa<$s1Cb}J9CJ3xmw#<>SCm1E!nE~1#HsMq)5lq=GZ2<%65(2ng zwQT-;PeFmwjJ~nQUBO>HoD9I#gS}3!gOVHn(*QwxAjaNmYM6>W_>${cq8&3@cE@n5 z8bJa3Me8}vM%Pf>!9KAnLif8rZB(qDwIpm76MDJ~-Kdylwu8iJl>h}aj5wmuDxrul z`c&1AU-7eu>tPswl+YaL)yo=i(WmH?!7|{p7uVMKoFx-(aD@g4W}qp>qP$N`F%ZxE zm64h5(J@xyh%5`Q(IKDEDHI66LcE4Qu!tJXX8r(nI3%{{A&5eV>lJEDOIJWMwLE+Y zF-aq!(nieCh(Qa=`f{mYRsu+7_plx);IIj|b;k=q7tbU(Xko8UlA^r?DlS*nsVrv? zj3=dO`lSX0c@eqf%S}4ZNf1Q^ngMJl$$eyx zpHx(qLOq&Ni-{So!hGrq8|8CLO~-kwUI0fpVmbh^eWZqb_-$5vMe>?e6eLpncNzAk z0*>DS(-Y$@_uxE+2vQbscn)K4R2FOh5XPVG=-s}R*#7z}0+W#oA@XNMoGO^=PTu+m zSH(;McYT6(F9Aswdm#Hhl+g_pKJse8Gcd6-F~+}vwC0X=?>DH+w3M)`jm8hY@FS+?K2JjJW~jf1+$9wxC!Ac3d99ihNogzM=dlMPR~S2;`BW|1D& z*m$0UfOLxW%E$Vr<`0S$5C;U~0D3fyQ!imfTM0Z0L$Z%3289{?snQ4`=3V~x;&hp1 zp!K1<(sY`bLpZf}2Se*CVoq|?nx9qWhwCyey8UmYk(mnKA0g#_REi^6`+i>d0h-NC zQJlwj%9DO&R&f+vw@1p>a;o zH}T>dX&O)&mUugf*6HiS&;g+{@c*y?P_^9X4}LGi)0#1bb0h$TJ zro{vTb7Q?mj8-7Sp6$wceT*6XN8E082 zpmFjMX2@#$Xb!y5_lS)RQO(aa9hU8yS;+t3hsX?DcPy#@|#%_>eF3(R76gbob7#lWs>+Y=V=V^rqHV5DmsTrsog|N zDk~Y+sN|h0*rU!X#oQ!6{p_0~@;3xLf_knE-axw7`Mw*(Z_tj<4auiUE*H==MF+<( zwBd`1-KQkl*g#Vm%1IA9ake3K z#m?gAfQ}k!&u?etF0tZ#s!4Ac&#L?p;M1;=Xi zzvJAWkOh5oe#6kxas>0;$hlfP1SYHZs5n19J;;+^eDaYFBvp`^>CM7$Lr=Ttp)tm# zMqrg};awlZm~IK(3L`dD1j2Sd{h+#iq(GoAGY8W&N+Tm^u)m5jAbFMp7V_lbMy0#XRt@sfWw!!D;h zm8O32w};PpM9B37EE83#g@)E52i=PHZR8^S8zVUA8f~K@M*AUyQ5HWOyd6!M;aajC z9Nz1!9-~uD6FwcSDk&ms+Vdt`&LDcU6$UrtmiQhGF49x>$0?cm3WpCjMtJCsLCP)T zy)h*e;(J!f=c)4@sM_dHc%?g$b;ukyb^ebTabs|!-z`9Dk1+vG5;NjKK*kw;CL8`x z&wz?Z!-az2qc=Yd=WnUK3=Cmi3wjaJ3o=nxcKgzOJnKF91x zHKAow{SzMvUM(tD4D=$((r>>?G-w=LU2WYBq=n*~D)8>VZ^JmzwV%wi|H3(K<>DD? zE9CfDo8pJ1*V0SVW?->oc}vi@PYb=Pp!ry982D$+@;!XI8&NhwWUki^`d?pA!0wa? z!t!sEVu@$9dATO947Zi8F^(#*-WH#%;~)3YGj6TOY-P*G+%$GB;OoMG8z&)Inv5$i z1kU0eS7ll4-TEVXX@R2j1`7Vp2_dazo^({{x|VDwN-L zz)X|-G%MYPb^s_A-{x}3ZvDTV(XXzgB7Or#nC%Gh^3clJO!!#Y>p)I6SHJ! zG0C5;rF%-ewW5H0&A{2r5a^bZNxZr=ySWV5rEbY&aP-+TOA~!MZn>(YC2i-8*{eIh z@v)g>(E4xuP*Oy@U-4xo-al$B%}Mvnj(@s~Hkhh&PhX0#MQS&`9^xH0(THY0^J*>B zrtQ4X;Jx}O6u;2a4@G3%+Hb9^MW0nJIQ?PZQNJPd;}*XmjuI45ErQt*i?@c_#0y>c zy`gTRVM|el=^!8BMVxf)aCL*5)|yPry@y~-_sj|)?AGn2dnTKL z3}lQ%qKFAU8wvB9;e38ho6iPgzmwznP=?#`!%Yeu! zIr8G%CZ-1Q{GP}S+@5OKDqKaq1y*>=V?F#sD?{?pT?F|W9M%ItAfmL87S`YKJ+-Nx zb4S7r8>@ISJ{V1J6v=&lZ2i4?Fm9xpL;5`QKvteAA%hwqMDvU|fg1fNjiEqdzB|U< z2xi#Yyb)6cL^alvgKYo{P7K56F_^$e2QXX2hJkuf+VpzyaT^c@3<#h7zx|fwSOo7z z=Aushd79XFw4fGZt-I>a)sfLV-1y%{d4#vbMuojt?8*o!v{$U*ry%T|Fup^tUJ_^0 zk#&uR(gfL{>Q}V}Y1;56JF?3-?z*<$&48jB$bMeToH^At?}ur<2#aumdkm9mDgj;? zfKWY5NFyTPqGlq+m+3<>UT+DqUN%B?-q19WBwVKw!UZ}vV$o>#vcp0GWfGf zui6;OYLb%0UX#jGVTidsQ8gr##RQ#)N4eKw;==#FM;L#9DyET%^^2!I(NQK<-;))( zM#M^!lsrkzhXPcPq!l~uy%wLHP~BjW3fhS~qXK@QQVuS{l zTMArW?%_XbCXZt$)GIgmYIotKyg{mQrBA`Jw=B`e3%e^TtLlh8stD%iLg5ecj+ZBE z`Y>{7Ld)K2B9oE5`ran!4#r4y&M~7;+WE#9W)|~K6gzEm25ypLT^t1zVHGO^%kM{< z0c!&M_ohzUFLv3F)@MnVA~BX=_Q^m=gfvLh8(}!jts>Lltakc?Q;e6!K+JdoM91gO zdoVPT%V}-l;-$I4%|r7A_ZGT@JF@T1+WG<+vXG_Y$??{JI}C)fqzF+l?mN(+I?C3_ zyKv+EBxY!sJhbVo3<9#VO`dPS3xwQqGQW+ye%jHl#hrTTF-hdL+q z=?0heh_MV)lX%ULu9Jluf7Ze3={PLX$4p?I^Rg=ad)QNDe@@pj&An6e9=me{q{|j`kjUv=x`zh^1g)fu5zTE61DYatrz@K3-wmeC~Ss8Xt;jV`LpQH36 zT%B(t*y<73HgbVKyypkL=^!HhmIu(vat!N{U;rox-Hb%Y0FCT#yhyxD^p z3OYDDR@TAVQc*>@gY9#fW@X+`adexuRg3M$G~Zs)$bbSXIn1VM&OAKywE*-^z)TJ_SNaa{9s|*71kVKLi2TRiEK{IW8+#b^n#Jtx zXO%&{L4JZkwM@ZB7Zw9rBzY^E>X+LbG*DsIOOClxI-(aeF83tczTm;J#sij*Xr{v! z3=I_!DB7k|Ks(`2vz%SU^HG)liNY0uS82-IEV$)cg6P@u*_wehV#}IST3!qQcB5>W z;2Oi7S;b>RbUCYhXc5Sj!)YB_IASm8WJC?!X)tDFL>PoIsIG!yTK@I}xuK9%Jl`qU zh<+0--22ud4_*WVrv}0D4U4*u`BaWppqa`F5^e}jNgsu`wLpg~X2^OG-WLsh#yMXm zIuGbqnfG-=pQYRma~`6>!1}y5#ugmf3yek4iZd&a|7PZ7Ze3@}cen=X{fXmYX60e? zFOhhCP|swrr@0DX(<*LyVFvUx`MVvStBfoeaIYn?Y)>6UjSyHxVBwkk2Ic((qAuzB z3b8r-Ibn7B6Q<6RYirA^1-OaQfC`rIP_T_P7?~96YVq*f8J*Xq1Ei#g&)Y9{_BDzo z2|Jqm#Gkk5)STFKXHLJJoIi9P8u!N{6WlNw00v z1iwAB2YRb`SDrNS(C^;0umy+Y5+;1E-P8pb632HTpP$}f++v+cq@KxrUD$HyOK zyl=;p8JMAoCYI;3);sBgG`+g@pUDjR%v=RU=sBha<$5%me~on6J|HJe_(YhIm0@7r zsA=IP@^>_bl9AY|Du!Q@VcaN^@o@j~t_APSyZs(_2 z3i(t_j9UzD=rTCSW*BiR;dz17#rUXPnx8Z#nM5=ulfs|}CCG8e zB&r1WM%c+uekbBeut(32ZkU!mn0Gnz#~Tlczm+F|-Ypi3-egJ33r>L0JpO~N%(+6> zfx)&<)-^dm<2@%-c6Z)EJ8VmSJ8X|5B=dz&FM7H38u6hHbw;=cIB{lj*{hr9?t4`% zFqn~g3YqdXiSdT!eF9kb0OWbH>}c~N%WTCB6bc#Qy`XQ>0|M7boy^;sNqg=VOa|_; z7S>U_qBxY;vnRv}$Ll@E^J>STb^63ORU6p2e1kV7tiK0gzwG7_v>UAd{{O$XKrinT zoSHJ4%<&dS?9?)f%y!dz9x7HoE>igH1A<(8sJe)fX{%4gR4CZ*ElW>eDd{qK3a7ibcffSbl`?d zC)A)MsfGGystMW*mFy!oakH|F4&=1t2r`erLhmONISgOp|HA?ZU}mqO5qPy0Mg{Cx z`hc@Fdremtlz%rL0xWPkp@W6nINf#t$rzx$xZ%ncU zV(KV20}`Y7OnyGg1ejz-p=R88ra^Y{6p{ zI1yytsL_RVG18TrI8N5WhISAl)%b_Pcq{J9C+rW+!)76j8>U_fOoL6zhqvi}dP?L? zdn@Mef3~f#(l6YzMf3SbqQOB$5S(eON!C?JGY+GKWgdqw?gs6*OdQ7qGFlilI7St&A`5+;!58!8v*aIA z)Aa{gSMvk6n^?2D`+Fe)Jb8;jvx24S3S-9pJrpSgyS{Z9t zrY$EyJoo5PU`lCIzaQ}VA*LqV50p(jLVS#!Sk+smC(SX_D?@pypua!v)s;Xg^vH0_ z_v&(?56(nX6JAfNczIl;LsclmCJtlxC)_IMCIM}swD{Ql{HEL`z(2Kke+^aUGk=~~ z(ppk&G<~a=IL3SdeiOzVHnwdxWz|ziLyD3>WZG5fQT&F1B0%W z1BN{&X6H$T@f&3vq*U*i1>d-iBRsy2W?003hx+$g7B3sn&y8Q!CdSTyX~0vdUPp>G zB8Ny^4NP3#!ir?Ge){Kvc1@Oap;o3ZNQ%bP#0=f3 zO5?oQGS{(UpBQa4Cu zcA|-{6=h0mh+I zUnFzMq7=pLAu~3W@HhQ*;(sNJu})3nvmyk)*N?Dml2|52b*48}cNVErcn_sr166HR z*hWCkn7&|LDFzMaVBoGO?z%?Qa#PRZ0D_z&Jcad+_?jbZQZn_LNSl zwD&2DI%n@a!uj9c$7MOB7Bwl`sT4l$ynSmrUB?Ewh_ZFc%v^D?;P;-6&Y934?yqMK zq4*KQd`9olmph&44(Ztz4gJib3AS2y8F?l5${b5C2dBLqTU#je9T%yO0WV9tS*Lzt z=2I4>-072Ipx)eMlG%}genzTL=HZy%0^ipRfBZ6yRzH&wy8NwPU#3mWZ~fl{^el-r z{267R#YUfB@3mY;?)iff4Z8$KfIs#nVUnMqLeLPu9@Pm?0??;+kQXt3UP31?XjT|N)-5y zTT4jfQrm_U*@MVg`rT>D6W}7KF}w^&P#bK2K$+@${6F+{7)e%2a1&suUx^ zhd4}=hw}n#^f+WxE5mvr+%WdKQ!WRx%9fSBxE#%%WiSQ&A|%{VJ1u9vm(L|8azz}) z)I{nHcv8sUyD~(U{L#&)=Lv09=TtMDK@?EV6D0xnBIsoGH$?; z@=YA1ygW!4yiM(}d}i%M;tS(J9g5?N14L0=1#_IO;$8ex@|z$3U}~gZ*-MVUVT~t( zV6JCj#daq|ZHn9s+3wEXYC=%kg*n=?K-`5gPIu!U41mpjpOJvU4X_8RX?!pqTXQqW zs_>9W_`E*&X-4`wL70Q1`<9=R@CnnLlK-PflAtDAGO30IqSDvXIHDtNyT)Vo*|{vy zwsr`@o-x$3>KzR(-Ps^Z01+al%z#|YNjmX=pIYMM$+gktfa6*#-ZAt64Yf*&IuN1Z zTrGV#nK>V}S#G*Mzau6LpdnJ4LWHuRkTU~MU2g6%F)->_ruwY0W->o`3U zQ7`DgXRT6_mRBb$1+%v?14XYYQ5T6??707;i_5S>QK8b0-Q^Mu)+s@|w2VY>R-=-) zVe&N^_Y+FVYzp5`Z8os!DuHQr9oZoqSBjWAhaq59>C$SB+)tHfBP+z+O1a<`QJlVl z^D@4BE%v)9f0lPfRwL~Ojsg!9t6KGkE0m^l6Lg!}Q4=fng8dEex-WQ0ly|FE7v>hd zvGIwE%_jTuU=||uI;>&vMgP@Germskt8z&B9adDa)0x1~*+sp&-rR4qxyyE(*(M1Jp#*>yEWtvg;BAA6o=S+Aoc9W@`R zj?izJn=jT|6xKe{Jv5ez-r-qDixH+c*SZ>eS*o_xWPecTO{zvKH&-glnnDyeGs90F z@u-%m`Sdj}IH})|Y+wHub$oCS*)2#7a&4yJ&Re0lveZ|~=jz**&}r?T2?_lZEO#fG zA~y?xy%V4!Pt+TG*!_P*eS?3Tffj6Rb7R}KZ5xekr$OV!Xsk`r*xA^PZ8bI;t1_7$oqh0#pc2*8`8m6)MH}cy ziRHrNi()Kb;N>AKd2u2vRdH&e^gf#VSA=uY)7sjgmS$U}X6l2UvgrItIL*1odlb^A z_&XVEP$!DYbA9il*+c)wyjlhI(Yz~f{v#L4ltMvqEE{-0LBY?zIiQo+4R7HP%qjT$ zqcKHWlyzLW$RWe>$5qdwPxD^kV z<1Un>Ny$|rCqb|r9bPKR3snWNYtS)Dd?i3!+>8YKU5~YxJx&=d)F)_ylGFwh`ppGt zKio0MDFwJ6T_2;G&ghDeNT*nMKG!aEF;W-dj^p1?Y)bM5oEbw)jg5QwCVW>b+$>|aUzKw zZva+FDu!a=;W3O_6Qem~1TyfgbrA4WwP=PCkRb|jiB0q@CD_U2EzH#EGzai$NH1AT za@?&sYdQ$BiR5cIQ+VJ!6DUO?g{fxPjhyie-G-1f6bv2cXbnPmo?kq2W_A{I4@KZi z*tq4iTs#nDWSN&Ju<3?f)|1SwL8C+7eT}8cA2wD7oL>T+oxAGEk(%*4OjQPR*lax+ z37)nh559z&C94?)QFapDpT7F45+VZ3gJ0ETq5N<$ugRgK-H!1LpYQHtPZr2}rH`Es zq}Dc%%{JT8qR4ty&LN8wm`ED+0C$&UEuVHoUbZ0gd9wanmkFDXUzlPItqy!;Coi}d z_JD=Z&jCN56zBQ4+Y^B9SuvvkKgF}{Pta8v{Xdiy_0%37MC`z+Pz;-FmkNkGjR`J} zzv##QTqn*G>zS?2-yelj(Sw0fnnSa`q~HxEkGxY~${DK5v)vbb{Aqut1-|*GC zh?~ac3&j_* zICmVUoob}*oCxoY0lkh|n-5rZM9aLltDWnSzOg0^_SX|^JBufb%Qg~C~! z7xAYu$mg35p9!SNAV9{`wZ26{4BSreEHC%Vc5TY>hjsD@MOXetfI zED$!46BPDA4}`yEl&zs!a!RLi=>GrI=>7_T5cbttibe?;3PVIOvgLl-(wf(;gi6ID zZz%@lxkEr0{I;os^=QrceqD)iKBM(l`Fn}Xu_8SS_o@X-)m>J2>KcrWZmS{;_a~HQ zdS@6!3}!NtTx*K5jQKrnmTAG_*7F^jDbQTauTiC1_1<#3$thepOqGFyh_uu(l3v z%3b=hd(6c(9yUFgM>E4jPf=#8USYGT*a$c(OE4G;aWCgx;IjC&#ai(z$KwZUORu`< zCDaM4m1rd52^ThY`zvqAF^|`O(Qq@r{s0G}vtb*gi(Q_wBl9a_BvO+b@kD_x| zuD88S{$D_gOs2n^zn3}+v@-Yn6VKswxXW!1E(q#UhiydBhYN(hXDj9Oj5IdO-@DH& z3p7Qgo<<>|F&F$=>YLv=5!>Yq0(p=|vY93#?|EVX3RCZRS6q@4%e*Qx#{CXG!;UIp z{+VElfwL?Ba9LwQ4QXP`7>lmiIc z>dz^-HL!Q767s~6bP}@%$<_y4ACnjbGI$%2>rCfITuV>3?|jo>8;DvRjf znVeU*34NMb;(;Vjps*K!q>&gnj%YNZmn;-a z&!Jg2-O|#qr(otOnMd5MP@yJH_&f3yH@f#|Qxn_F^0)=^$jmC1r8c>amH^xmTGogl zA1 zg0(1TYJp3HsDfyHyM-eTh9UW^KBLKb43R>JCgpolAdQ9nIq3&??XF(8*AF1>x`XfLJuc%Oj2yH1tNsrIV`{e zxn@1<3Kn(HK8SxOfi{EoSFD(XZ3PO91g6w~3Og_xO&?@58e$L=I|Q*Ajh*H7`|qf~ zXhVfdJ`+bsas?Jaf5(eVBoQKxb*WgzqS>OQ`0yu%z0pi&S}t|Zf#r^GW0$JMeSa9e zB)K+jnLg4V2&up>Mf@(9*oo&WT;`kG;g0V1!Q5uOJDb#{l8WYlawA|TODA0MsM+p7$i7@EnYs0Un#ovsm{2Q^c z7RX2yA!Fo7W5N^U>e>?1i?w?zZr?qx5?!y;wWMvV>Wm$)gQ5cTmMYu-kud3@rGPl8 z%3Gp8E*;jlUdpF-#4+a+-*fW=XNPEa)Rl)2r+l8CgG^09CwLENNAWI69`t;F{g*towk+C{5$zrnvNcVz5Z3U^ zs08*@0Cx{s=xPC0-dLOO&YIs>HPZ8{i_>*QhULQ(UBgT_1xsj@dmyr3)|j15!lnVM z3&%*de?<;9G?I;lV+>v=X2lV`Mub5JfY1?O7ohiW8iY|s18ue~ z`~gmAlOEhHU-lIqH>g0vUM}198iy-@i`^R{8jswrSln!Z>f@LLb##k4Kq%OlQygYZ z9?ZP3A+-v?SyKL!y;}qBqWQ{h)|>27myCdM%KS?*eb#8U2qtYrR1B&Vw-aRo2sKSh z$`I1t1-~puIiagZyNoYZ6=k(1SCMz^7B_L2*tWsYw1%wrFN`K*j zNa8pfH6Tkv;VGdDH~f9sSp3y@UI14z^K>`ha5+1!ARX(zrw(OxugtWvL>qT{VBwBm zWzr%*LSM4aJGlME{)1WUIbU;S5HzpV?N~)+E3-#hGsMfV;65Y8WG@^fDPW_WbE*J))QoBG5Rt0~hql8^BAXZ>l+Ep% z`aLF@=|B~l9y%T1SDgNIg-tg3AP)%M)T16>gp5JIo6WqZbB_D(G`x6@ zvxyMWs{uic^cI_YGi4AB`GUK%WiaUQA(s82mSY}&>0}meQ-=CRbrI2mIq&E8*L7yg zAnsKKYz{~Sdc)bTphFPC>EPDZaOxh^`_sta!XKj@hOv#&Lb>Vqzp2ucBo;v;QMqRx z&6z?@@&_2Jd+?Tp;Cyc)R}g~B#>{wXTLYAr$RFj!_Vu|7)Be1eagGNO-L}lOJ!`NuV7iQbBZ~5gXKb~cn!}LvQC`QLB^`X+&EqxXHqb+kTvvAd zGPUHL)slPctCz3NfH5Fmj(3?sNW;%Fo~Is4DqV9o*3n7&wAa^(e7)UH?Phc)j>CBu zj7rB#>K5!U@A~*DEvwTfT;w2PrM37I=LwVZe_Vh*wY5XWCIb8dQvH~_%#~0^TaN{D z_8hd%ibR_sg9w9X<+i3l8b;0o9*d8=kKHCWpVu5pwWhfO@as`WRn7yyZ4;hP&qGla z%|}!QuZrX7#nH>}&;K_JJn2^Q;b(Gg2rO0oalU0XdwjHiv5x7;C;R~a|EQ-%}om&R|!4cXNlzPu!ye!b&{t3Vfc3tg1IPMp@w$i{gF0)3-!*DTbg^av75Xq_ijVG^+|A{Pe>d_h<|-azt+d(be9;d`I;He<9jAX5j{nG$EX7V#-AtvSSbO02eY?s3t0G1Y{r_04;y_r%xqHJhOx--gZ%P{fiiN>># z>9;muJz9ZY;s(9dIGj^bES;*JXEN$e5-Xku`ufxil3{^iST zrf2GwcPVs_Ml~3+%^aOA@0kK~+;mg4K4!mR*(#On?iMiWv~$66o&;TnkO^#GNd4jU zM#3WIxNOFP7dS%|7x^`AdX3^x9wZV)V9PrKst$~OceRzFIlpVgF~VV&foh@;abQwp z{=P+M6inxc2z#8=D_WY|g7E`y-}gzMC}ix1PIdGK$W3xstb={-|NeS2G|T4*m;Trh z39y?8lxA{JmN$?Vz^`Z)f&Fsy82sl+eQzc+lekRudDt5qBo79P*HJY}zNSH0-^@9r zzN!vCycCcr#}Wr;ay!emf3;DnY|RG4aq zqXXMHRb(SGDTS+dql+i;^#7?_{MChl6cU5aPNh>lXWT56N%?KN_ybr(7V@(XW7`pr4h6sE4hs zE;~p>ufL=V7%#Qo0QIzy(orzO=^U4ms80XQ26q2(&W3$}*IN|IyIkBa*qAhpi>WRW zXY~kjC{H{-ecr%NbG49n_{WA%l4M-wppTP`Q=ZIO7L(ZuuLZa1{mYd3uKrOV=}00* zN=vmfAeKKqot>)@?=Hj~4e)Y5N!^hgmpU6eT+t+(DtWJXX@-ke%Wc=>GmZjm?$bJs zP|U`FBa|-=v&P6<;T9>n>kBszTlDG1j>WN1<_8O5L8)ifixbk}TR{U;#$%;^;*YnFW|qj*SglXB)&xhXN9tjfQyba8jooEb@zmEFOvgo zqEY+2=aEv75jS!S&P+q&huV*O5zPmh{J%O()l5{zoF6MC2RUO7UWUbvOaPw*D#H?niob zO&XI*Dz0tP*x!M&i_EI@z>J($xPe-Wp>B$ct!h99;|Mve#*&rmtVyU1rFKW=W#Q}MJi8W~x5`tZ3OHPP$Us;#4hX?9i((D5*8BU+5 zMTlrZQ++m{BoN%;es=l)mUDkkqo#Yq-*`|rL|J%4YwYnhOlwxZiWFeJw&#^dqMtBV zF8N4BEJ3QEeHVvzXH@QY;X8W8U6V?GRDZfZL*7(Zo#8519=**i#_9l0dYQ0acJL;V ziFJ1PH}-HcHWBRBviR`#dAzs}N! zwang%^lp}zz+7E$U!nh+1DD^d6YcRgcAh>XsE@GQ=zT;Ziu*P7-$Lz$gSq21`RR>< zvdjTimrQ>PflO->5Y!KPR%>r1+#rqZNHe_#3<<%oO^)d~JdKl4{Un;EP{zkSleE(+ zBUZlJj!hck9+Qb6*S(JT*A&8zA$RA}S-TEdE&re~mJGf{7r=-yyNR&?2n6!KvohtP zxy0ouSTTDP*kdX0GkIP(146T_E0?Pk8%vezu<<09wzMoKOsgcnr89!}ZASq#3_`{m zthgYInCv5pn9hp4z{CuHPQ~;mPh?`|g?U z0-nP5{Tzh9LdUtjqGR@_k5y7&Tw9k4zZVNvSC~lCKMq6&j~;iZ==D>909+pZ*wf0H zza;-nc>lB`*MKgIonzP-#5>XeTwnyLr120pGsDywJt2)7l8$iV`@5n|{rjR0c6I==f&`K2=|0CSUrYc3Yz1D=u#J9^6{-I#}*6y1J+m ze?0WO(K-UKm7ASk@=*95FU1;qpZX>ifHcNN28Kq2EJ?KA7PZnBfl96`Sc_@XAMqJ+ z*|I#SD+;V(W{1XkBW^Vv;1gkJV)dIcNX0B}> zIdy*z-I^s|P^LFN^ZpnYgS#?2@2K{!l@uMXYTl->Kz6C&mmML4!7N}$2`I}_GQeiI zV(?)M!U7i_J09)rNjk=p(fu72IF)+2(d9e)x;>uk*c|9#?@gJ_{Ik!Cq4cyJ!^lC* zmSPnpUpCuCem@1BO%AnnyFa1LBQJsfxjj5-sU#s%*i!odBVe1#QAXd|p322Lx(YX; zThG{1!w?`oy%_hv$neeDB_an`UzN*EDlG%j7iI*?#<D!9@1*RhWK^}DJ6R1?)yUzs;NAukM!a+&Mmb3K`E4N~j) zMdJwepuU^x#|M7LW~^<7{TLC4zU0TE(?+r8$fcaztEF1p*;`*O(1pn8p*u6- z#wn@|$*KnWnT`z#skK%8flH-Sv$>Ezq|_<+sGa&NqetJXcua<)7qdk##cS2dOVFQn zh8#acq;UhstkwqTpp6?>msCqq*e%?P zp&GNxF^d24ZZ%99=J-g`bp8&|7*egsf*Gnn2^+LlTHfq?`)Q`ZYNG2=4;w5=^OQu5 z)Tzj3&yL#nx6XJrft;yeMk{`u>{R(pm#+o;<(@YO#br`OPuTaWQJ4yC*7wVlZn3x1 z!dbX&lDSdCX$P+wP`J~5(~+sTRM0tB1QN1~ux51VtNBg_rDFiLaCdY^y8I{VQc}hB zj`)ULV7K-_9k|CB#wM?kpI_5zW0W+o^WJu+H)zr^z`hwjM}WbirzJ{bi~}`XSa@t1 zr==Y$HyzHSE1`@7#4fRLNRmJF7gf?fc{fY(x45YK(g^OBpK`3toLF-&im{TYatT2U z(RAS`{oi8yV*HjfiWYq46@Md5XNRk7r-)FPIu`t;4D0!!s^LU9szgZ3*tJigL<1?A z48A8sClOvVU3_4#bE)p4#GhpiT>@H~ENUN7nuskyh)2|>V-{?e*}JQvgoFfmvOo+u zrMD-tShaBGAGFmB@Mype-;cQoyX(8Z!lggBtA4TLJ4s~E3KQx}KWH#X>qt~%50=Q$ zm^ArI)RzY^9r*QK0oNd(ETqzSe_)~c0HoaFtkYfk`I zs?pejc7CBl5>?}UcEq-D(W?*5MN#ydoO$@5u>i2+3CprMTFn0>{O4i0v6}M!+h2@* zu==ajg|McQ_ zF%sTTLh<^KJ!RIAz-5hF}lgL1tadDxw1;#&z zzc>LH%M@dm;R7fHHX|b)ULq%%@mRUMqkT%zrx0BXu{12ULgIZa4(BgKMpm< zDC&<6Q}S~^sj3p%7>QRz{9%I`HOg1$XRUL^1yB&9gmqb7y!OUcmV93eJ%+K+ZqiP3hfm+taa>hK*+e?v$M(mYojp2h?ei+8q#{+6zYxGs zxi($ehpkHy6D|}N4b3w7`&)8p48?DYTj?MOal}4otTzSCy4#YWiZs)R)DE_r9~tZ! zCGJ|<4Z6Dx*dgfe@SD){_|0@VDuncBTA04ic5PAe9Jw^ndaLGpyN(o9f* zvo#tP3d?E|gID(lvF!*WT_QyWcB+B!u9-Q{pz!KSeP`!fjo{G5uIBEpjnMUL14 zk<-7x^{3m6t~-Aej@ru&YC21?!E1KtvFZ}d&8fccBN1a9jYTJF)_`I-Bf2xFQm(O= zv-WxeUg^AeUK5*G<^u4jI!>B4{c`)F{TrW<5YeA$WfHekdEW(tBP9n4U9NN0k-As; ziZ_+uNd*TfOLB8)fhGgzc9r&3Vf$rr#0bO3n@WjPdM(;9J}~wX>7{F{6q6}~zOXei zZy_YDYWOTv|FXeYNIBT9s8rp%Xqi6EN{Cmxbm$o4bVWl}Z-`e8E5Iv%^uQ^JKv(l_ zt0G$xh!6ZRJT!8HV-qcYVL3_-51$2=Um%@}8XtMU0^oA27n5Dp> z5^0$K!VA1REyo&5c4=hS2#35f;m2*8tUV~clVyz2z;5;Z$C;nTqYJ5+s}U3=_dhfM z{raMWdlP48&`)F(VeN`Y4ve;$TkPW_pC8{dR>n{|FckB3#)&&4^7F0QQW9Gwdw3k; zm?H^2j&mgX!}kWTN1L#Ifpb;c@pPkyyt`wX@!@vt zT^?4A5ux~BXWYc1#|=hesjwpCx{AsFJQUu!#Bm-=>^ft%1NRBexi=-}Kc^ECg3be& zeuanqwAI$tLh^(y1WFXST~QxFyc6j`qVK4$*)J8(64({Kq`pKU@r^dK!ACK*Z;zf6 zXOG&y`IWfMpg|51;nH=Om~L?y8hZB-()Q9Oa*B|7d|OJNzh;z7{#TasXC=1@XTLmO zoK8EK3OnvR%WCyEK;q`516TtigNA&1PfW#z{v~q45yflTtg4in=g+#ZApG1BrjbJx zu||NV;I%pV7M7!mnVhD!z_y@BdDdvKW*jAQI>_>Y>))n@s84>A56|%}gMG=L0oB$v z0=vet#_n!mhU@WZl`N*(OM@>vP2c>R&tL#&$SvQVd1PRCoR?>4r~VCDjAcZ}>IE>= z4yttZXoBA?6s1aN8w5DLCTq$fu5p7XX$6arZE{8j>}+UN!q@i@&F;IK85V*~++{~c zrI?AdBAJ}5i<;WT9;f95y{)ab&Oz2zU+SEw(hY56qXM^v!fM&mpAlef&KYm1Wq7H+ zj2h`t$^{WeFF+0J>4?8o#kz=5ouhZL<7}kgg~$f9ab?(uwKbZgG4_7IwWuPgQT&y+ z!PiX6K-6U23|!N-I>J~_M)15cG5k=-{<6`vA+R)o`8NpS{>Sf=67=$0KYPV?Fvrn9 zPk(ECK6=zxg6MLzX;cVC5UNxY?_fQZH)U-R#)Q9;U&J>IHXfruP6FXXgM7D=ucNzD zwjswgU*{Xa_!G7{ozOL|@*BVh(YpN_oTL{^443wS1}Brmi7q&PPd~&MHRX-)4~L(& zeU7N{Ob@{%(HZmbv(k-Ld{Ms`(jrHLeau|PoL|*Ho`gHZVXZnfRzR25Nb}n+D~DBz zMLA4sdR86%Fpk?4$1%tih5=oT`;SkSXl`h$u@;L>$+FD}z(|u4mMWqrHta25XnYN4 zlF6f8>Gb*bAaxi$?|f>Px>e--EyGQxi}q8at_ZKY8qxB&-jjJ;h2;KAx@>NVCAT|F z#9CpenMU)`O2dLQs~FZTevQ02VVp z`0m#V9APvcOS1YX2IeB5Q^Q%W7av*k4h9;&)|E1sE06h)7gcvVR;m$;^)_~Mp50$! zZ(m`i{tm*14Ji;}u()E`b(6H#Tg3CSgU&Al>j91KQ2U?!BPIKP17n7_b8AL^S})z6 ztpn#LNC7O_C;cs_aJivSm>`qVnL7T2^%B^oQ;_Gae%Xs2gh2I82pMc;g{jM7&xf z0B?(;n9FD!7aZD--vTOIY=lm*t3gL&nhESXU+^08)hK*JE%}%2{r#;y*I8!1uRL9S zG1=;}`tk_7{~~@#7Oa9PQMaXy5N8gOQhq=@K5{o_Cu)^zZAjJa91-zspP)ZBAp^Sz z@Fg#uV*6^e5PkcJD@Q;RmVZxh*#*jfnCwguRlW7Uw#A)bJot&J^oxW8> zsT6S1vdXe%8krV5KR;&*knKI#?=_B4taY`nF4*Qt0wPm=gOzB)L`k%FK%{;(3AX() zq0JlCU~^rY=v57!)a03{FW9tTp`q<)qA@%=%w*rjXKffmykc4CyWI4XC`qQ8f%WXl6;^ za1C$-B6Q)2=BckOOXGfnMqSEH&KF8q&F7K8Z40_CfmYp4{?p*hhgtpw1x|mj8pi%y zc;RSzoipk~F0g=5vbkEs3}e*~tC*Ir_%sOaJV<)3R(vRe_v$;mYi{dD_KrhubG)zJW^A%@3zyZFu?z_0vp@Zo1W%rN-^LNAWoog zzGxutF%!|_Ml=e8&n(>P!~0V%?A5h;=tPfx9T$EXu+B~mP|RXHL_Sida2?c-!QAJf zj2mU=nsG1_JT^21&V1l*a71zKNH7Frmj+%@^A#lTo+74y%q3vJtq*f+?}+PIsaDtp zu|$39^p~+n`A5~|l-F7TbIW|4IR{(1M{CU6Tlt?GFZ84V{IWf@Jw!}zifJ=KYLHT_ zh6vzmRkpD7E@7?v#-=fUOn;(b5$)y0oywR#ZC4~)e<>h#yqn{G{73ob#Fue|{$RNJhYb0yqYZ$A@8$Ri;S&bC`788aWx}M0 z(1E-+*^5UCvn~FV*_PhIsg%!IB&IuDBET@E{!2wdn-zLX5NFnBk@EeyETao{ON5K} zgb8d}T84gi@VmwS6kc?#ZpqD9T*fv@$W;IzzHCv+s#ej5LX|LY)680bYk5PTnvUWn zXt}pA($a@8!~=kZ;y>jKfD4ZG%<2W0`CHywnfNKY(WxR{_2rWEM~X#OL7_Hl#dQ7y z%Vk)N_w-{kg=3y{g?SSFA-O9oyLrN%a5Oj1D$!+6CZ-02N#(VyuHR@{^>~0lp-?;2 z@wR28RwV`sQ2|D`#E_I*I);985b5elOIGUc_HcAYR%6c(k-=~cv^UJJvLM10(kP(T9PDHnr-mppsxWu8?>2Yp0bvx7e zFg1ZpGNK)!?BV#b>9mnC3e#LZ(S%qxEDaMeErXv%;C#A`@Ib+N@Y%-MI^$G0?d=@D zWl+V(qU0)vyyJ5n39JudsJAcm)3mHLtk=B!jx;>spERl?BF36uSy8(?S|BvEK~W&g zZ2rAFWaPYre;rJ7Dx|<>B-aUF`#pI#d8E-%DL4g3C~Iz}(#grGuBHyc!fz46&}E6@ zS2$eHoS7YnWLFw zt`lvqNqm6C=DQhHB*0hb7Zi)WTg6B@+UQxeQOvog{v;nCuWm-h(CRO^@Jz4%?w;Ym zTxG#4cS#v6pI-L4tMVcbwID+yBauC|3VpU|^WJKj6yzk9r2eNe0cEUQ(=Yp{Z@nuk z=JvQ0`ZS$Kh)+g5zniD|tbR3zQJ|pDva2;lvZF;K>I#g{=EIbx9^(j+Da>+NdG;YK zocF7K6`!)3g{6p+*j*tmROT3fap>g0T%!JI!DvIV|6YgFMrWMer%J)$lhDAVc+(_t zU)7OKejmQJ)Wam8SaYVh4a#3~(BxWX#sgV2WSNzc$gVw3xR&N+R5L`;E-DZa(b=`` zCuL-;JY9S-#d72tdu2Z&3ubZnwW!Wm6vpUUQ8L5l)q$MekcHePqn*2=du(4A+8RUo zD@}>~n)0s_$#bqZB z9<#O@6y6&tl?-<~vc>a2kpNjH98OfHF9gypd9KgtkG zS0l0H1$LdXKW7l+`e}9y{h`iA!e6~C9`{r}8smuSkvz*$)E14u6DBfOjXxv688e?s zS9Ks)I?9DqY|)M9j5Xc6eEA-BP-ykN8;eUpm$t!jEZa8Xqpl67xKj-6ua165!xK#5 z^E8nC3xw(HMpb2)c`1+3KW<4y(|bf-8@r9w<1i=Jl(wGoO9!N^w%5A4ekbmrViD=9 zr!W>AT$mcVNiX|~t!9LE1r-A(mMS?DY343P5cH+#z+oOz$m;Du0ABQm$i+;hq-p3! z6|^)H#%7ni8e)hCI8?MKz9$zY7N6R!X2y$o#f)8e3WH#~J1Mv}8yf81rO(p^ooM&H z*wY8LDkLwbi-cI_dZ#~81>;d<24f(XQXB*&43o+$KT*Zew-NN9EW;E^Mips3En zewGfe3~x%Ov*jQqp&mM+^O#m(R|IRCvmPh*9A-sSX4*1$J6(dTp@f8AYjS(?7#*=~&?Jq!|5+wqyBrWH!T|WC>TA*Zd*rGy=RTN762GJ3}L52+{N0rp2tcxUx*q1CM=DK zD~Y8@qa9{WVWw0exPW!(<5=2mM13Du%p2-&1y)CV1S?I1&gJ?%5dxX+mMAm%|J9)f z;4WZ8d+rrlOuec?5PnayPyA|GTApGe3p znwSovc$)D*!>c^J+(z#sxMJs1N1~Cj-x8cREJoxdcnH?#jDEj2pHxYA*w0|5m+(|N zC+@qyEgh9(9RE11qx`fo%Ur3BW%3lAdh`;o;+S}l4CxLGc7EtDqzc)jx^Ul7W#Ru)H3JnT!SV+?1?JW?_a)} z8B*_w!7&n7NiH{buh2m*{xeJsuAqK_zf?3h^``q(;P1M;J1|{8FC~toT~HP_6knqB zhu~$$wCW!o-8m_vlk9T$Tbtx=M*~KO@*h$Y3~46)JTVdpVXYz0E8MVJeH^1%__U_P zbT+%|8v#W5ZdXp6b77=$ybe6|hQ(|{t3R+nX~ryOQ!zX&MawGdha~0!LZP%dDPMkE zODzJ91!(5!7%#jlUDJ?@oWC{)%-vj|cnMwri?tFF=tvRALVQN zlFE#`{7u^Pvq>#>y3`J}5qHRcmP@*ff^SId!x%|3>PNTl=>K=YeC4p>I4O*G*RQ=# zAFlJ;-54$x!N238M;tlaR9Lss4T7f`K4y^bMY)_U^AlQY#AdV}xJdesbIZu?tK`uT zoNNGO1zIj|1<{PFWP@{9*zc847>u4()_z&SumwP5GK(#C2=j>} z;2%$hqW}Y@kDa|I4eC%H+Qzv4bZ*>oDHC;(OuC9LSdOXIu=@6p`Z>G8g*krle~ z`-vqX2grm$o1gVQI{3`j%$K8}?9501VP`#Qj9=x;%wLaFh@(+I;~p#lPiUw2+49O@+IeXef9B`$csPSEEHWZHVa2ChW zdZsVf*tKK3m~BH7BC@`?_F`hQUQ(}ps+ zII?V$MA)x3r5Ij%QcqdSAmJ{kXUPMPWTw0omHOYavXzhanaoVE>w5`Qiq1Vv5vf@d3d?M-7}`<8fz@lqL#k5<%rAkY)85aoMGOLyM7^Q>R?#F9Cc`gKp-)`?qj?u0^m_nX4!fJ!7$4)IjD8=Qshg#NjPfpKVmXIgMdY>z)L| z#`Uvo8hK5#Kv(^s=r#&-ea<7%AjrZwvaVNk2M65c5ua~vVbig@b-dckDsLNv=Dbud z0k-s2!!1S$GiI$c5QX;G>x_f^{eCd57#>JbgQ^b8Gu&7oOu0dCtPgOvyfa}vy+gDc zS}MSCRcx$j9a6p(k{^870$l-`Xjpm?x5^2x+u#BY(ENSBhtd%V-&TC%b8c9hO?UrCIx2}B;)y?Oq9XbA=Cn3)ttgV{t9vGwc_%VCR{q7*x5rDyP@HdrO5x<>yt&Ou|OK&Hu;n=)Ghjpj3#4}G? z#-EjFz45JjZ zz}`6hQ$O&~;648pyN$x&2J$UNFl`Q3#lVFIs_8SADK5yAl$7U7J_eD)ufbTt-;ZW2 z*q3~6!}E7ca4fSJD6gL;C1fzB@g_ptq;{pJy zc5wRwSRgNRyaSbFjA{QO_FlQc8WbwZovYX0$j2Rw%SF?Y=IYJs7$=92hA&@ZYJ2~G zVK70k59#kx!{aL@E^Fif99J8G?ICl{=8@l6t==8#x$Jq(A>AXV@@qzVTmi88} zobLV;rGaTfD8(dPV9r-itJ+Ewm6v;g>2+_yGq6JJQRsJXaa;nr!Pc=MK3zQsfyAX} zjufnETG)N6Gjn`Le=7)F_DHtxQhgovR~hLDqEmEl60SMkdgFUMNAyHmvYuyQ2^`l@ zr@7BBB+`t;Vz=B+e&xR00QC%VwIjH6-G}uw)t%$J2O}{%&MVfT*R7Z9mb{6X7;ei; zdcIGbC*6BZHIh5+cDE$-pAjQEx6ACGwxhF&xo=j3b7H(VuE!^LcjJ*g|2h`qY<&(a z+1O+c3${AGdsn(|vy+~?q_;{gBGT`zya6IxBTax^TBXb zAa0AKfOE9QYu(Ajn8<97H=1_5leUCIu^hCOBx|^-d&ibs@jOWm#b@cjAKvO;R4s9? zTXaxay!NterZCm1pp?FXNK-1NixOWXk>s|$n%NuoCUdvBYXxmw9+USN-XQsXyvAd?>lycJ6mr}9I=+M8sWw#a?zG0MKQZ@d4zZgHrp(O)japnI`_VsE)5}}vjqy!T4L@H0G+Fra?G%dc-LDFtA-1c&wlXTUhq&kchrc5c2 zvSSvM%A{tAUu4ZQsRfmaEFc9X`ISR0LNz8Ol5~U>#?XrbnFfX&7(V8H;mFSg$#o_* zQ&@z1Xj5)oimep5&3S3Hn|9K#loe6gj1zCx1-Y&wtSOd?h#AnbE&i|OH)wh+mcops zqolDL2Cx$N6y|RM+fH1~TJ?Yc(50!%jC6|av`0pUJ=vV z!aBQhWqo>4I2C6m@pSp1sVZTrbMOX@xt!95?IQ)X;rLiG(Zs_}UBl0jbUkcu$S_)9 zDVqb5TKFB*CNKb62OONN#j!#kY;1qj4k|B$i_!D!>oY(!A7^5fODlQlj`|zto zn*PzOpO#}~2yFM=aoL^q-y5}GaHEiNOl(uU)8Z5)MMjBWSr1ccsu@C^4y z!bFbKk&5ej4G&JIq`zP_LnqWv;@JZH2B)mFsQyh&pk4^VC#lGhkx@ef-j17dNq?C) zFAgQFE9U98Z|ivAJQu23ijeR5X|Qil_~Qw@meM9=FFq#lZbm1D3|;VSL7hs?REhCfmMktMwK>&j~#z@;|u2CXx5s^yvHNTc2y;jZt4 z|9gIw!&4Bv(Z3`&Jy^8d8iT_HpFY7t=aE90*ist8;xZ+KED$I1r{eH?_7v1_K5m_= z_!&;x5)t!-AAh*9f<)9Gy>i25|46!zI8y7_e>2l?9e+evVI7A9DXL{7z9Enjt}~A- zV0JgQK&t5S22j{T4?MD?;-dxvhwjviE#KV>$LS8>lN>EreZ9pV@4H`k&Hf*qM?=cK z^Kb;*3R(0kd1lM$AQk_$Yy<*%tp}Fg<4}fO*Y!SnMdzkgj3o!Bt_M$;258Bsu%w6y zaV^|(fB$9cIdLA7%K-r#t)MTuIdwBm7@h`rPhw4G!A}}q#ghWI;HRCT;Y*WabpXfP zWoamwbL>xfgk zNtnSN%JW?4dg$5I?*=-!h6a4|yCBA!G7sf_jl!6U#}4v&+hj){BahuU!Li5_z?d#? zjfqDRIjLb}-Y|yv5k|Phru8bf8|hs^zf|| z8In$IxwwJ3>~Fp|GP}w8&s$~ zahL7CKtW8QmV<_;6vWu?6%=;yAqnA0Qq?zErZ7;;P}vOU*dH0`j#GQBM@wF737`() z<@oDugZ_xIbYrb9&*|=PKYU)(^%5WZln}~y$JnyvaWGeSTOP?g&F#sJzvIopE9~$R zvw@K9&ek|*-Lma;8r49&*T+1~Yj--CY4b8a#lBVB+39rkikPX2`TA#|1r}^W21o>7 zbsj8JXumcBt7Y)+4o3PCn!s!K=lyyk5Oy;pcNLGSjoORbHFuPtZ7fF! z_X>v8PDvNxG@|hKV+i*iQT{O4?K*V-)!Qy7j?vM7i;m*Q*AsMa-b^-sGI^L{cWu(* zakS%s2^;1n0(RhKbyf)mQiyX>wg_xBLYF(dHAS7QP@DDMaD7Fk{qf{Ix^p}k>2Pb4l;JJQyQ8emxCF`jVPJKl^`c*&2x7*@EPy6AC}%{P$%@-?Z*F|;$>OO)@xNHbrZN~_DOT| zykVy-uWfco?;a&*nrr9n4RWo29`bw7mcD1-T(12t9f$xuuXw!{;8%2fw=mh**!6)U z0l%^f_(eNleK3!pZD7EYOtxK}7fl&YiZyHe%F+!56T{uCEfIkSJJ5!Mjw!3yDW(z< zR}8hgfK+--F0!}8dItzfQ5B#Zje~cirF#r&bjKRLIU76ch?fj$yI_x%N&{iW@OR!| zqSDD*U+nfT^3mu+ydB0M_nhVT0!5gtsoFv9>7p?DlA8u9hRy>^3Y5n?de1EtcO_Gj zf2JzQu5)np|1}Yw;?KQnYe{&(buq?1VS{t#ey3HERH5l^!wn#@tTobhY%`~==Fguk zDhv~!7sMtk%lUDw?52PxuU&JRb5i~Tl-&$v@_HkuBn+jOAdiO398$NkBDKaz7(N= za}F*8q#Pxbip4A=L^u&R*g?**{lmz9v_I6Fp6y}gnQu!m{qCMu&je1jlPz}g7pH+073bw%)-ySYnVC2gn!K;w&8FUp{iq{@u=V&~ zVN%!K=P{4o@6T5ozRxwg<3O>&DyAMFf z`CjvVCdYTP?YixHy`^{iLX+eC2%Boj#68RJNV10z8O{+x65NG!y74ajTbDOOE}3naTc6VhK^AU;6ykN9tBC(rbZbruP%!Stu_G_N236Jo-WJa zytf)0c-eqO?k`h>;wk|*4!(^s&X(<|6bfu!Yg9+T9FNJ56WxjI`n4FwAT2k z!!pl4=MV84K72EMG$iysetZq>z4gY1df45ygPj7eq7-bg%_=Wt;fD&aQy)QrxHd7@ z#VTSJHnbU)=pFs+^|K#bge2envU9)aU4L?-z~GREYe}g5OmgrcQ4|5BNb=%G9WGO= zuU-De04jc4mqAprs!JBtjkcLsD?qixhj9XdYxVYc#cRij`mB>wrHd*-u9J`r$K*LB z%A^2XWKnh6hk3L+SOAhJjkXij+~P-_8OEWw-QTiQ#|Py4g{A7UhCChDSeWXB(-IgI zdT1`*8U+SXO;sylF+@Y*Wc+r$(;W9eoEXx4{1VkL-S*w>$FaELsSxX)_1Zjc#-bzF zzH5NF^>SpHPZKW;^?5oP?+HUt%+8|0-F~@@L*cuk^WJvC`%QL|<0?>ky^Y8+Yenke zeONNh@Q3kFoukw~Gi9@R6l0t1Tp>a78ZVzA?EkocjRGTblI278ohh7KgV1XvY35TK z@G^%WMc$G;*y_KGCdz4jzl1NDu8z|kDdrCl4O$PlW2zM);rUG~(Me1d>xIiF#*d7C z&4@%Hkskk=|Cw19?(Qkf{4&KAP}>_k@c;We~9D6_IHW-krQ}3AWXuhzr4Iz&a~Scy`5=aa{K9ePQzi+89=L zQ=z4Kk&Ex@0@neUsE%#2e7m2%zA;8@6dJoA1!u>B%?EW{ZMN8EHD@>l)XSRr8xdG} zp*&{6n3<~n0D8Z756K`^N6W(sH7KNOH?Ki@O~V#>=nR0ZiGdF}7jMHpGdv}eHG}~Y zW~^5-#wK2=G-X^%+@uJ&k|@4Kdi^t#v}UF8H)zS4k zdt?6twh5N&fPp`~8JaqY==HcVw?zwxoE`WSYzXcoW+b|h?k4}%^69YlwiPa@lFzFu zqt}C|{*O)K0xgx3l$DjnmyjQVdSh2(z}CU0?P*9zWS$(6sRB5;*Z1QYsWWV6C^zGR z7HN_xz(V0!%6}tB63?h3w6K`^Z#W8?2(s&a7kj@y<@)xCoaDUEp4YqwU>^fNL|Gj6 zAN3o7%MKs!2>9)X3b7rZ%bBe={gHHS_fJ~eFY9WbN4cMUhqfTq z-*=ewou>*6T*%ixV_!eN&!tVX+^3myKAS$c$KyS(w{f?hS!Xd4(!2|MwYn`6;zw{D z9h4a@9IiW%o{bVDMh-h3e)ynMxO`Mf8LgYq(A(AECHOu{suCq-z<Ji?$-#4vrCWnJ&J!Pnlx}RM+~mFvWm`HBfn3z-Ul}{@C^%2I`ptt5S&>k7g== z7|TNY9dm)4@{$#;&M+3Iq0p`e^EY(=T{)(uYy@i4YM#W1B}ELPl$po{+OnN-LPua?nPi|0yI2)(IMK)| z4#-tf(2IyF=$(VgSUqxizcfW6a&BT8kFcCVbDC1_IrOB^Ha~CrPiuVSn%rVP>8Qv^ zb=P_c?qcZY7ahpt5aR#l?Hg>=ols@?zdA9~RAJGpE`PetBfT4Ra7cluwWd&(&Drey z7edeTT0YZ#@T3*P`{IZFhnx2`|M_!EWya_0csLvf2y#FXH=pAiXM8WzbZ>V9#lHS2 z5yXNVm^t0dKA=8ScrH)#o_s_SyqA=Ken27K012|Mf4(13J?FS*O&ks5k$*^G1?6o4Ce1A`tqC`t!ao}PyWSGg!7v*;oMikUHC=N$U z3LH(4x@nCjVWMIpWLx5j@;V@;ZPQ+S*R`g({C@QRxrZ%i$BIQQ%9#i{avv0C_0Z zs??~}9@uB$)Q2V}|5a56YAB?U-O| z5MCMm4_nqU$njLMpGTA`7svT#rLWBxxRjOuX_`XYWJ~v+i}(X*-rnl(4_YQmwq1(pY7Wh zJV8U#viV`+vvS$8*;!Zj3aqf#V)`#7?Br7R?k5!nOv#MKQr-jikCPGJJH!>eXLp>G zT^`@}5}Mq%tdpMIOmaSoAs}Pox&2w)v%Sjk6WCm{8gEqIKpdjkQna)^bcgx&COm@j z@nE&?%#uV};Yo1q^6xvq&s6Gz5BDq%x`~1MB>QMSFut{#s zWNNtcRr~5si1)Z@w(0w9IS3{N1~Qlk{e^ZuAgiwbz~JiWO+Ss#6>g3)7ESy|O-eQg zJDMW=J`o;Zw8nMEUQ^~h3&(HJCb|gD(RyE7t1uL=WeK{?90XlTf_xc=JXQn*eJ+a{ zmlG=MPgOCcA&b_MXx#!pGtB1{qmQoPqI?piux9-4=gj!D zCmavzhxWFHOsSiiBA#xtj}c9qk;d%5^Y0DVXMr4ej<@G^#|gf0&_ZZNJso|`6_?#^ z-~07_s2Z?+m~-0g{d&Ll^m9HsqS^o?a=G@~9iS$(vl)H91GIKxjO?l}&jQrn(nV>=Fq#F3P2ai_iC~Y_{WRGN ziVO=T+s#28)2fV08w|C!8W0PnW5W#k;2)%qZ`hr!BodHD`-H8xqD3U1RL4PBpMwn! zN=7@Qw*rm7ilafK^Q`BH5%msXN~0IRxLVyy`fEkwAls&p#Aa^*QkP%-NV@*^|UK9?)Bkt@+7fsA~3P|Iq^Ok#olV!aKa!oks z&KowX?FN5!WdEacLhOy!#|P{^z9^XY>61&2uP9QWn5)=mTZ`dxs88wiPO zgJy&Moz|T15a>_pphZoyT9cwh;n67GLbguix_kwu!MbNQP*zi&!9wGl9IQzNag0KQ zm`W4unJx`IM)uVvR7}LXv^5KOEt#Ln=mNc#O6!x210>|8PXjg!DzfrraDPz*Tv`=h zYo{cSRE;BJ#%bN*5WKZ>l~V>uCnauzJ`))n26kGKE>AiU8|a` zqAiyVm1uUY-$NwT);Bj>zqdW*YL?I#cnGa*vRa?32mT9BsT!6|P5A~^3;v;M$XJ!{ zqLDJg275Fz1fEcY|ACj|toHp^*SI$Wq={2eUK`5Nm*Una$06RUIBu0f-op5e+8 zI|*6lhQz0RhJBPpUMfJWv>~Kk)EdjA8z44Mj%)w)X4SyHx*!~2PQm_Hy06G#k+EcT za-9Oe^T#%t@TnUrB;IIbK;41@`H0h^)fc z%90E0f|Q+f*d*k>6h5p`F5M5GzqUO;bem|6g?+rIJr0ejtFh`kV)`( z4kw?IQs(Eq51<2kIPtc_nj?lMcabYWJGZKQD4Gw0n}yQDroG2h6=V`HZt&w6)KPj; z4gsIlQBqLhkvLk%aqNg1QGzyZ)g$c_j{)k!x$h{YBYO_Mnfm8mKBt5d88KY#Ppq)~ z)z(ZCLxKs?U56!Xcj`JtD}OxOneu~2QF$Tr;PNrS=TLYHrqv8?T0+#gT~5?#^qD}T z93^_(d*i`%{ja7dG^2=Db*n4DZ-XQ1onHQi3CG)1M~+0clJAz#nI+j~wtSPT0E?;$ z&W8HNc2hn$Cu(HXaNsmS+S0OuLN<$^?T*TBzV2}gi_Srx<~bYf z=LUDF;gGo%XF#v!DU)Z1E?F3vEtDT-n1<>&?_OFuW6q>M% zuKvt);8=9nAG7jGC1`1B*+&1ZGXJ*iU3iQdqENboMBWwiA*NfAKo@a=D?wb*MvudW zDzMhMk%~n1J^)jDNlqpvY1U*r^Q>*Y3T%CksPKQ(RwJ+gJx<1QCJ|jDY2BX#_P~jM ze?I!jXV=nFlhFu1n95YI`?S#T<6{>GwvZR35K+e{b10NK)82F|rj~N?Otb#L71{Da zLPwbzj$p}!{fT@$^=J`Xv>)D9O`WuK6{B0+*Wyx|8kqUoxJ!EW1*W2wCX};BuYL^H zp)+g8AB@UkrMTL^=(*uSn4o9l0wc|o(9^oXoMWDQYcIxvcdOy3in2?skeI=?rR04& zz}6y9v*01N-dFwwmy)WR2F|ZGZ3q)qtE`{L@A1Z+3R|We^ZHgrWCTd2?n5J)u>-|Uc(=v#k;|uc7 zs&Sg&KrEmIt&tjHfGTziG^ijkKmljOvRQ^v`dz})R5JoxnF#~dxap+{hQ~t!=~~VO z&-{eHO9eARqbOy|ggK%J8-H=mi$%BDZV6tq<|b0dlMR|G84nX#mpi5$XXzW0CsgbI z9QiR-D@`4hErZNq07Mil=<0A0C9FVhor zo3sH24YZI>e>QWz1+)OgX49$`SE7-^Rojy9aghI^I*}UC!C(lySWN^UQZup(;V|N{ z;7v()vKcc7v?@bZy5-90zFaPQ00)E&(4~S!DU|lfI7?Tk8}o1d?CjP~FgTl)u}cX9 zw07%C-Ajw{e_TM`mDkK=zzBn06zvK)tz4O(`Hp%|#`>=S!zuNb=oHU;cQhZ%W&^o_v$j<8Mx*WU}c|O_;>&D99W0V9g{g(%y<;Vu%6ulDf=; z3@cFs(l{TqoeLB=YCp_Upf@Dih~Pr@hYx;#(Sd-s@aE^2IH)gSFpS9-E~-uxr^#W* z$wBj1!W3@ioU1i&x}$M{8h!|dyXG~^*=O8Cqx-$$D^lsC;UJgE!8YxH7kyIhTtP6P z3nWe`d%~s|Qf=~E6V)kI0t2(!k;bj@;#z6BjeBPU9lE%dqgj~%Yq(uIYqm2bq^r+g zUOh40t`sN1F1?#Oj&9D@9)e_wwP(?3DE>PV2jlhK-I$^jnYnZp1fjp+#-)R-Zbkgk zaQx~Gt=b(4LC)5-6eRrGa7H^8efW(+O}abLRQ@(j&T_#>k(m2C(1u1}=4g^fLZ_Q3 z66F-!L9YWB`JF&)QNp2QP7VF||1I5Xl@2Hisa2AS9lK9$z8B$~ zRG}LQ?DS>ix-<+QxQ}0eyZge9qfj2tDDCWQ9{zRpvWQ5b3^9&$%V+T&$oX~2b|%lU z9@5j@B>$;{8bcJSumeso@iU=K#D5|CPD1CQyyv1ud2vP*^#G&CN=xo%A~Pn{&t}U> zgZdK&b)8jqk@~rqe`@ki>dvpSaRZ&w7==SV_EE<4;IxmcQnf*^n9$7A&*M!N&}+P0 zv^iq_WdyPgJvxKW&p;|~TZs0+jH2*Px&Ap(kR!aGK`<&B5<*vN38UGpfLjcHhBF$8 ziwtO0_89(AY+DRhLDiAdzMQ+Qra-q!JUP((I^?4$v{Yvq^k4Q!VXG91?OafRXe8%6 z%73ABEQS?SY4#cnb_|=_>)A@dogiYRKPmh6!1x2IfUM?~b@TWymrN7%$nIa!_ma3Cib_Nk6>I%AF4dcApTlw2^5z*MZtH(TihAaP$H7r~a zjR(IbaP>M5#m;s7std6^Q9#2#7MNT^`>sNRLM81Gx-X@jx($Nyv%M<(1L3|f{Ozx; zm)xjCxwmGT{h2nR?eTh$Yj--EKb3d@`Hn_(iCI8{b?6)aBm_@4-|y9^^i$x=P~f*z zFG3f{@k06@@|CM)MF|kCol`o?8@oc~2<>hNXOtS-eejamO8>mW zqf34ach-~WwR7e(A>=bWIWeeW-A(R^PZ0M+Z}s?Ie>u+ZJQqaLcR$Qa5xlR~Z~*y= zc{3n6Uw7*U=R6q7f~t0Q!93l5S`bOs^-w>e`?LyNoHCad!|T@gr;Q9-)I>|xSSMbs z3o9-9;EX6@LUz`0k24ohL&6(S;c$LZnlk1-9l=oeGo@B=Q8hqSE2dzY(j*0;@_7IP zkBOrcPC$!%9uA&`o|@M&;N-B2^C3ZKP7{NO{vac(#nDx;ctPg~d+{$zKi=zqQZK>s z;RO39%D0j2;6IaNbR_J(b)E&N+K1ceXlWn$H*~9)6SBjk^npabdZ%c3QJSuG2_M{QTB?l)R-?kqsleQpmPKQS+0vQkdbl@tcD_!D zXgb%`V}D;Ey{et}Yl4`vX-~4I!m;z$KLvFGg<4J6y$MCxZ>%dtoKt1Og$K14Dit0z zkG*l4+`F!wd+${4x4DJ3+H!Es2~MAjm(=c$;+2oNik;7#p3uKmo&dC3e-=Wgr{8Nk;FdCHSMnoPR#vh>0|$W@Nsn)B127-sKYIg1Y?iMTQds7mAF>W4C94kGLtbuw`#s9&tx?K5_HfS&V&k}1Eg`SNWgoccs_WBvC%ys| z3<4er_#1+F!2&75(xu@|lrXC^X(QvN!;oOwZ3O07jgzbn zFwYM;<8S7_d`10W7yZGQIuC(-F9S1f!zyJVPBJGHytlVwci*flUO%0P!Bw6 zI-?r3euT;Yqb>4Nx104~u@{<0t)q(`5!^?>;yI5g=vEZo{1A&&0>id@9tH>RinsWi*!)Gbwhyo?1TlD!y-0 zv}{!eI(NbKo&))Au4g#zkve3cmphFiV(@w1LRNadSbg6zNW@}_rsxCHY$J3i{HA^k z<+<%54;$p~4bOVN*k^}@s3Yx;TGUT;ol5Cf)%}BiHH%yN-2k`Sm0!tQ4x`vaE2HaTZk5UAt&30eJ^>i;Ns$YfK&Z zD!TxN5{~s~D2HpDkXy$u=%M}vQ>HeOt{2IUOWj-v?BIv}M(;~ffJPN-Gnz7>W3)v?UH>cty%Oq&R9&7sF8dDaEB{{mm0mBAo(!ZYxh z-OTRy7vZ(|77QfqE@X<0aL3TE(a!4D(vjW0=MmAqh)4ki*X3Z>tMBu4O~Z&{Y`ToA zYLjUya;wvz24(pefl76(8Gzxt@jKX_uXDYx2(=u?Cr*pBNCFpyBa88%<&~B1S6A;9 z+pl3;0&cr7_BAgq{CX^3-ls-;mb3JI`;r-{HGg&qD;E^-X;(836GJ^dDTBTV;y1*mCz8Y7scn)FJts#s9R_Mfu+6b2nOieVx)DuA)^{Q}y0*ziO72k}1PnA`|q{f(XB* z8XwmfqH5zTZWUFI7dILrRG3tdb%JYg1I_{wSY~ZVmyuCMklLjq`#3^k2A=E=*g@`@ z@rJoW*?_=A{Li8PYY@^DSUIlvUT@n&O;|C(-$4)hr?4R9bk{*~V;zD9=y@oWhzmC@tWdUK$jyb`ACK6S#AF zL?!#kBb>^eji$wfaQ5&Y^1pF2F(*@0eOi-sn8Pry%lB`J%JyMr;1%{Ng#t(9?9tiN;YSO!ep_nHg> zdllt#GZ&l7K_Zcsq@LjP%91A`b9crUm&yPRYy)Q6vGvFsZMazcA=L4*bcWUHjG?Z_D=c5-D@@A9s9;I;MM=&dT zUJ4!`Ylkn(^q+0U^wbeIn=nFVg2Tn|$;9xd;aC2A$$lbCyGXVY5ixpmV!Oo8Z9I~Y zUuvHd&D*DSvECOwpDo|bcf*iZJ>Lu8FXb~nPq|lJC*o+!K_7ohc`rSCPTsY3gPhZ~ zDIBzPeL_~VHSAr8SE|GI{;9P1Plb?{urY8=;*t+x3rM0&90E|YnksP7e|ttRa$8WD zS`*xLvY~p2!lVZ@iy^%IiBs=SngClgDNvMB!RlA1b5Kpm)dVUX91}?r3&|-cW2*xG z#|4nr@O25G-&LLb;+!%2apzDfVR$!tNoa|~$|>Ry)=%kxrjvEu1LKB3P)yJM2kJj3 zQ#I*EEr%~QDEYA|1%PyT^A#mAfX5P~oqH&6cM+1wK4uGxJ1HiVUe}I8L~2QxKlHl0 z%G*B(bSUnibG;m%ZICJ8Rj z>o|??g_!TY0j3Vr#ck^w@PFfK-MH(_8qWjY)LQm^qtJ;-RPkoDahfKUdBak1Y~jqa zH%>r*tPmJ5LBNlz6yz6B{%Qy;F*|>w;Lb3aKzR>ILp$YHz;AA%#)+>gOhzQ^(n+n&p2 z;LkzKhHMUQ?t9!lop1lq&cjUi(JU6E_ZJ+`=f4?XXQ3BwDG!XVxqWKsx$OGB*|x0t z+_!GoIX;qe>qqrH-TAyJPSqLTcnJ6De$DK8eAa#J=q#<>0I}w2Z-fpz8pxR-!pP~& zsxpd~5v?>gN2y?I))WPtt(*5SRObUWZ0Q{*8nKjrWhsa@xT}eDFn+^<<0gwXdB;Jw zmxYIL$pmyrt_-77Y5|r&H&DP6LH&-WX@%L=v(EO-bZ79<<>TeH$Ngl(FSICNdpKjW$2{`mZ@}T)n`C>&eu{=*xJH$r ze8jzBSLdC#UX~#H;DQK}xzaEs?nnxGrH$2t(BCHqdqn}GPdDk{m|rlXytYY!tN7zhsU} zzYe~i`Lypv-q&6kqxnAe5I+ayK3^(yA0dHugh%!q=i_wKZ2M!K=S}#wESDFV@}#5= zO4mY9sIp%;)baS*6HMblZ#nMNSt3}lX8ZjG+rHhR2gJa^vv z5Pf#L-(!S@GH`4=Zwm>&N6&TN*He4;(PO<1$vUAuPhT-{x$SmG*FQ zM(gfU#d+q|{J%+l*t*MiVTW#^^9m}Y<_+wB?=#C!SSt?NR$F;<&IKJkSKl0J7&h~< zG%1g0t3-1Qr%%YPCmb74+pwbA9`H&5v8>Eh`K`8RJnV)QqtQ`EA>eohnh$pb}jZOl*-YFNJ;XXgoNNSUxgC~0kcHdHX+?jTUB zs;YUu&uWmZvHCi89-k3$+ichS6&KbMp1+7?)9DKvYf665u+#>GDEccdA4^sy-GF0M zGesD}%dkqIjv)`H#f_0s6{>)bdcJd#lb88eFlx0lBd=iFFvg6DwB(bD1* z0Eb$x-mVv8#r&7Ho@2p~j_(DQnSYKEi&71Kql&iu+ec=aQM#^S`ChTAc~3!6FaU6SQ_D_MLKJ zawlXtCnJUSQ3ZF+IG&l&5!$ZH{T4$H4u@H~B{vyr;w}F=!8|f+=YCC=eoy$m&s@8m zYykV*FuP{LUC0>T8`z$kZ+^j4-s8IX!pT3I3=fdG&uztkRmS z_vS~a$tZ$)iWB}=2b62A!=}^XoPk^T{cH%x+xEapnS;ASTL3xq=0rbA3d;9yvG^k? zF_O@j2&W{0P!u9ej!ELApLV}n%)et>MjIgh{vV8LPA-8*tQRfAA&(*IBV??Fs9!b7 zS4X>f_q&)S8XarxTNoy->_DnJ-E4P79sEeN-dVCO*mcag-votx7Sr@x(E5tF0E`6% z8D9w38(?-GTSb&-*6F~*B#g3?d<8C*E2(C5!xysjpnc~}ii8XWVOEIFQ!gva%~}`n z%sLgPVZON9K zh;W+@Fc$-1l^;8EJ_O^)rA_KTcsPn3NvvfBS!QaLR);DJnV}E2Qy~KCz$oG z-xc?^LRdOCGi=(3U(*`2itTt2K9(l))kQ7F*M2Ys<4;7Hb*%?9B`P#QK2xeanZSpK z0yS7Cw;NStc@8_bst>95@rY3W|8wOp>jm$gz#EnU%M5q+Cz3!Bh^AifM^e$y|fTVKYp`_l*yY`v7An=6n0)-5!T%t3^bqWAC8W9RtZ29Fz z!egazZ@J+0Ux~pAs@HkBh*NXYn&87iSc=B6f;vB}!U zhoH+0bU)73TBLXpFTz@akcVfHHGmTuloCH8V3M7~tE`5+LipwmaDPW5o^W0^efiL| zD`uKQ8fQp(Bn9KM@PL5X>zzu6tWn7kbowS_z%Cgxr$#I@_BTM9w0l*6-hGZ;n7ZTk z4Ttx2Ozw^$D%Kk=J~ptr>b`=C-vV&wX4L^r<3FpM;4~@@^o(I-9d6m!4IE4$)r*Vq zvebRe1QUhaqen^`VZ^RU^7jc{M5rK$XiNNaPx32%Y_Cdj0r@FzD9dX5 ze`8XZhi*f|oZ`)hrlDlWD!DbCAsU1pIRqAp1;gfx%)pfiGS;FvsXfeJif6FMB9)XO z{GJi!OiiVR7Fd6r@Z^UT z9lTR^%%6OiJ7*(D@7(d51s*h8*VxrN;aWJ{xw?dp6|Z*am^_Y-%Ge%J_tGGPO@_0S z*q);_VRsiX@%X5WH?DUKpG4n>MA%dbm>?WV^pz7KHRLN~f|k4yRqvGyWu=3=TgZ^l zOAd`QHNWqkF{!m@#-a%#!LIAevxq#_ebW7Y9jmHD?(}liu_>HYMW0Kw%(Is^VZDY^ z#pp=dOv?JlgnfeZ4{SQ?6C;=kI68{GvtSXogGF_nbPQvB=uLP(iZK+kI)$X&U;|m+ z?~N6vxs+cSQV=u*(+R)=L3U9N(X2@4r@*#@Edkg6JaZX4@y9dYf*siRlHdvfA$MUT zizVteej8fuu`a>Or~;0NlO?j~{doaFpis@1=MfV6D6RI#;Z-Q_TXHT0%V=_2-Y_v; z2hB?^g?M)rfIs+r*slOMFDIv45lo%0!I8ou_O4eMi6`hah{~)|P<&MAT%I5UQ>997 z?^ulMP~sJ_7rDa&Sr57@^P|*3>6A)cNm8sft%-V27?F+2JBc|Q#%$q?%3CZ)T$D<4 zp|)YneDR~Qg9q-2%S}|=sV&gfdrE^%oawwc?pR_*LB(gTSQakD4Ct(4v_;^=yt$?c zj3msN{^mp>gMZHo=7qgFz%A%M$@fD_$m;hTyr3{l#;gqc4(^uw7+Gp<+?VZJKuy_q z`@fRRr8vp$WLTiSMg!H=+uDO(*kUeFS$7uSo=2iAiZkP4R9Qe-L4^@0Z{`o#wmNUTZE{3S|Uu|6E$FFiKJF|gPGw@~hGZ5jx-#SvKy%jIF>|Jv*Qbh^+ z2xW>FTtyAy4;`sRXNXhHuR`QEibv_$4<(;_uDG{2JKP^WvNzM0#a*juoO`|=r8b*acoT=J&moVaVOtz=8caJ%#yR{BIIidrz<~n4iVT ziiTa)jYeilh9$Zn^xIRT#d7kuRx0bfW*X)bsu@)2eO4tN#O^|~ zkEU~ejJu1rews8+V>V72Hnwf6vE3Mr)ufGW+h$|iwryLJdB4xS@BL-|fSEIA@3Z#W z>q7uM&byDsiu>0jw)Uu&+5A!*LYOd_)Qv(ME;!Xno||kNK-{>NIjYIjPCx{1q2}3c zIX42pne)oQ<Eb&6ril-6iI)7#>b8IL^5)w=Y=iziMpN@cR?QGgFZ0*gy zQ>#%EaPc8u;I3RzH>oc{?TO&?rjVu6&c9duEIB#Rt@;VC0B`3jMfA9dT&$wtWkn=PYoJkp8JLSp$nCUG zPW-}D3zr4TI1)HyP8LVY^!s>W^xO$wkDTkWu72B1?((LOrcGuX9oc8UrGw9s{oR0U z$uOik%kZT%;mMrlQ!}qPg&)K+_f^RN_Xz%{mT?NZjb&)LjGAii^*!ImLP6NL zkMoD>wdLjCe6g}jwntg_{oiMO&sxu_WQR0A8~y67SD?$67eh|}n7THvDU{B@-wxt5 z3CiE&Cvidl_AiNs$!ve8T_LYrs5M+p=__O5JZNQ>`@bax2GzR>Lo@RSIz}E*Rx>-D z5Ze9lJ~dL}XYZGUn5OuYKB1Fld>hdbX3WVn2e}7#yhEXct_E>MLRt@e0JhHudCVtg+NJ0$boXsYm`Idyx}CAHxz3`I$0)5kDjGYrrAZHU(ag0YH8c_`}6_hYqcBoe_8+oZAIr_c9{A!$SQhH%y)y$ z7ffsR#2a0-wyo?4QikNUvB91fjZw|g$jOY|r(|&re#&CBs#C-NEkOl zZYzsLX(q}ZEzEBKRXm}-?oxte_>uvAukc@cq1am7sSs4=SJSTuhI9K6+!eflz9T2y zeA?+a3y&e_wyP_(7Zx>~Dd0HyGe(X)XK;x)9V8%!jG8H2o04WNtL*l{9n4^#9TlE}GbC4ay78=vk&IZck744Npev?hW6SNj!l?7vN^-uR4lUQAOU zOwypM9H#ths^0~CJn15huJvgI)=r6{$gC5^Mq~)8DNZ~X3&jw`&R#)Wm*~%55L?kM{A3#_ID-*oHVHfsnS4+&o5(= zTAOwxzFam9r6G8qn-GICw_Yy%?m&Xu_^b7kJZFNSBe3`F{`2*PTZYf%%GmeXJ8%%^ z`^(za4aany=6DxTlv5L>d{xBf#OB?qFzOCQc&#YN=3e94F#OR?p)k2ZC#dOM4_xgy ze+8&I5asW?3wgoswa#*}|9)R8(|ZX(M`7KPaeN{CxutZ23z6qct9#iIo3wyA{3bV8 zo{UQ)Pz=fZ_(9vhrRoT+=JInm4{hrNe>nT?r=`8j z?7k0Ntv5V}UDAeK{ceSvmu zhpW2&j2B6@a^7Jq>d%~)ZD!}H!CsE*^tggFMBqB7-S2JmL{vs|gZGZpwNDY-XxH(O z6zzL2s&p^5{}rNTs%^RaE0sRhq~eht5~9R8%w{jzw=Y>A@d;TGd3CG`$_Qb~G3H!5 z=Wv171+Q+$oClWL6w}@v1i_z^sq^77^P^%T{UC#|3CvH0-vkhb41;nB@@MdyGBRZn zj;!Y91S1#!8!~-4WHNZNS}iHcJI5bM|M~Y`hXy3eJY6bVlpRU|d6G0IH!t>7s{=k> z=mLj{O$_n5b)Q34iCsortkmDt!jCQu2jVXlq;r+^|+8u(@0z(Pe| z`GEzJnybQI?0PuSXD5msr@Wne zyjrnEKE1&Lx4BR4PyBQR)v$)0q}Nxae{YO6G1IW8P$rtGP&)0Ia8o&EXnxYEVfC`% zUNrrnu5)EpA2ouT3bd+Kx@-yAfC!EKcVnDs7_gRAIqN)~R(bi?L!sUbo5z|Zo?BBh zh{w#1gMTX2;nS^W#2W1uH+QN{ zLLOZ6L(O-+i2(?hZeR^3~7yON3y5rkL#>_sK6AOZ=oxx!hj@|AJX{ge%P z52KBN#g&=u>JgG8Op-;Wb~y9im3IIb5(WS=)5vBM+x;PDm)SUFM1|r6lN;RzL{0Yq zo%V8m!FW}ysix(cl&}bT<>GH#?iOWekCBA>O108x2@moxkyCWaXqWOEl@#3;7ZHd> zBgo|*J={B?Ug@I*!^U=8XH?Dq4j41+WLu&eJ{)^ieN^f!bwP5`m+qwo^s4Ik(58RL zz$3c^QH$4TPI42SkW6K&RLC;2d3+OOQSG;})qYV4zBmHE#b^DZXk$RS4Z^p3KgoW* z0R-JIR`qEh=yn_xyTV=&CQr=1lWdy)?3H_1S_r0T-UQk|1S>nhJvEm_efkTug1Jb+ zNgwRkTY*2?5@``jAnSA%Ev@T2q%msbitM^e9g(~1OiOUop{3DSaPK=v>-nSspkL_^ z!a)5!jIux$ZEFh>f_CC z2gg^qanM+;XR_$}hRj{QEec8K+;12um^)sr%@C5S_pseuf-Q@U3XlT}kMCmsr7 zh%rkhtLiV4BO|)9-zF1kZvMsmA)}fpz~1lPA2FfZS_0B=WZMU*)T{s~aMbV+Q2V0x zHw?S%rG^TOdq{g?xZThlLzod-rl`3X9Zi{6AW0-T1z()eZxIP>y?RQtw0=%jv_Fe8 ze%*mf-w>H_;o~mq@*UF=x>MKv0Bnf}dyomE=9&#Wb}~tM3&{7<@UB+%sj*4hXt7D_@{l8yw+A1hHW z#K*rV=JX}dL&xXCU8|jP(e!{08gx6R|4~52zw%-CA=h*OX6QW5Gw@g$Gq{bj|E;q5 zo}C~}JMFw&m&RgNvLTzA#S|snqWtHB(;O?6DuG#oE?YlFMIx(kk`pj0l}hY{1Bi^X z&~uxx4fX{?IIbHQ{=#O?V%HWuLdWKsB1jK?PK6PNav0N};zw$9Fun@Vp9iOf(CvjK z8xuYQZQCJ~l!pSeuCt}V5J4Gf1Ov^*qI=T=`xJO}>*Hbzl&{fsczPr-Wfltk_2ACPp7at=BZP zNbadr%Qe}D?L@sngT^}bCKy!Z8``d%se(qc#7?J{7Y#KE+@v4M4T1V0kndyTmt^xW zPBKoD{%3O`{`g|yo^(H?GF1A4D^b=!|67J_FyE#eZg^ ztza%0!|8MQQ+-@#!FYDwlrqR}U!2aWrApL9NS&{IJjz@+Jbv+^4h-o8|bXoGU=Lm)!&jZnzW2$yCVwW)e zS&XaYQyU5yjP}lxYP}`(c~$%$tC9r}s?7Vk4aEdid9@LYN{A{s*JVVPAjpJGyM+IG zdG*qn{T!N|HclYC6BR^xCjO3pT1gOgL z?HEnDR!M)bc+3wsLeZ02^|MDx+3JGwfbWn{TzLqI#+=Zj8Szjm?MOX3_Tnm(sX(8& zZ&UR2nm+k{8*Xuc9)z5r~W|$&($F4k3?Ok1!IIl&~1nck45sq)^1Hv-m`Co z`|uCf6|?A$)3Wk{OK3tqhwIbwY=B9a743VTpO@M6>B0O0AdYq2j&(WB1mc*83QRMC zZW%lFGVO7MLq%owXQaYs(~|Qk#>6FPLgU0yu@h4Q`pP@W$m1y$Q85{z$e6<+(X?)q z(a0&yO=CZ^VcZ+3t;df}5i}YRMTg;J^ibC)YUxWoerMIlo>B13DxZYq!TJd-ve>YY zau){3mk})MfgD8e_BQkoX=yHUjvOq}0pZ88HK)$8*H;7Y{R!d_K!t^zk9#3g9qkq) z($@5YUV~7d9Y$M^OFTZOwBAf2c4xR<- zRbY2@LJSXRhZNs3Os~>C9F^M{&~-f-Nj)^A0-ZiIG7ScsY?TY7(aVmd zI6gU9^&&0x?|C~sAju%NZJ=)Un$T$wliMNG?D1wQ_$to)(NZc?(@962%wP7Y!~WCF zH2qV^SxYIkdF$p>dd^w+EDa&#gFg49(CEJ9I93@uSO=|yY4~pCW&ISpPwfRXc((AA(ta3U17iK9N_1poh;^Gnt_!usr@xi z`J=Pbjd7b2pX%kq>KngXy1DyP^1|Of>Xr&gT=_jfaoAM!y~Xf*t8!EL5O~f-(RbVU zhqCSm6cWO=eea3KQT#gZ7}oY|x8!-&ONj zt|9b|=I6KtObu5A6LP;QiVOkHe^;31gV2v>G(sgb@I$qd2;RA)*{ZdSnPw*Zn2W;- z0;I#t$=dHiv7t@3IT7ye@Ua2q)U$psFK0vkX04({#-MpqhsW_nOp5fD*9e9*x1|?k z@`@PyFPpEJO)!d{RVG8xh4MsUhl+`R8XAaFZVU^C<(7AC!9k-DnE!g$Uf8gC{$7kP z3PC+kI(HWAwoxkGbYK9q!|qPAxFL88EpZNc7TdMnPJ%T^gsqv-2?eM9Hg^K5a)^-q z>s_3oavC>aQ5@=wUB&PJv;bmvzpp9>IfgQrSEBvNF_JreTk#zNCQ)1 zgON6ZZ?OU2$3OdX3gHO&Kq9HR#>5!9_Fi#e4C-G+?BjWJ5+k8I%xM$O74-^1{-+{x zyiMvyTd)Y~yDqx>BsQi;=!Y#pb+Du>0dII zomYiHP93RRCV<;!5zoSR>dBipg;d-mDV<93<8%ysbhh<;#J9fWJAvqXTKv(M_29zz za%lhB*yRqw@A3vE=a3!k8nDBal1h>Iu#3kkf2570j?hv2d@nmW_CW~QoBvgwYXRSh>BpyxB5FC z=F5@ia+BJkU0Dfb3azUpHDKhp&XTKR9~$Ne45>c$6+iAx#XsZvoHhBb;cq?% zovgLaGt8KI)Lq_g`F$KwAu=hiy_Ri(HPB>T4HTK**4pb=&My%(NPBWty1X=W(t@R^JgJh|8D-?@2hm{p7i=g^xmgG> z)){r6S!vBA$6v8c6-Jh9))uOhH$3_lp)xkpJPjAi=|6;+FIpnM^4_(YVfu>`Crs7c{vs z-cm@t`a!-lt~gI|KmAnn(k>`&>O7d(yTQHP6oyGp5yuzmR`8&j_(NQcJGka9;M`A@ zt;w5TX%y2n=PLAgniAYVpjFMTawcUXlBc~DrpEmWS-48BPW}^Dp(>`<3Nnh@i69)K zs=8EYRC=3o5$?}&OH!8PP5z>aM^hqo4|Em(5TQwRPm-!S`f5@Lxz3sR_Lhth?tv=% ze^84id0_-h!aS*0xNE&YkuBi1GPa%ingW34JJr1}fFPEJG?v_^4v_uF($xoYas(>) zpzGCc_-F~FeEj1+XCI2fktsn>K6Fu7Y60dY`*Vy$ff-`X-c7=4ia5fHY)oV+i+^q^ z-tn)ZG9XjBnx$uQ4T(#zoYQTx-GF!Vf19SP*R>x6arPxSul!gnxR8xR)x>HywOID5 z4_BoQ_TQhEN!%(@%bfl&;17DQ#uCPZ*N+6jv)Q0%$xD97h{eUlkA<#xJHyz(^GI6@ z_OFrpug>5T8oNaTnuTq$8?cB?&UJz=TAqsV0gZhnCc`K zHRthudl;sMhk^~Z{TI=Jphvc6nGwk1qC=~Zyu8^%O*v4^8OS%5`!)P8qNW!h`&Z)C zFZ2Hi9CzJS8j@X+^v6k`QAo`i5kvizCfnwO7$+Ps3rO3%;;L!Tec`yKxRAa%m%Z@y8d_brDN% zKNU~+JRd zsy|W&L?=`eznE;d&v`u-6qL$#oY$co6vAv0MpkJ>vc5NF1kXFXDBvC9kIu*`klx30 zb%T+V>9oDhTWR)d>Jk5wRm?q=-|bM>%UT!dsemRVyxdG>(V2_Oas~d&ZIo7K{CD;d zG+bt9o83A>fK|BVS9Sf&Ptlq_DWKTyN0wFe;3l268)9lP=z0Yh|6wvTJI?SL znDBi>u3gO=F^QGagn>)E&-8zobU3`sv(@~Hw96~;CT?1=genmi%qVuffJWa@=^yE;9;B4gse?vQhE%{&W(OXGAc0`I|a5|pPUhX(!z3<710r^7*NCH2J z{Wt3%4*$nD#0cRch|*Ow?mO1+tt6apCP!mXjvLwsk$rT6=^#K*XOK}R_qI^QCC0rl zb8rGMW4n!=l|S?2(w)UvLvl42wF!1TNzx5IjDn5U-pI<#9}A zo2L}&&a?kO*Q|Y18gtbt7>db|ti?^i0*rKYm}#e)JbJ3RQYD}y(2ylt>vjO5dkwx_ z`woC$>wYD1;>;k3u5P>JaSj`Jzu5BIp3Y_Uw%0yxN#Il6F_ek98Of)PkyjmsbJwK5 zyoG+Of4jk~^mIzg@jg$8g;Jw-HUIjx%wK^9-9oh)XK1pt07&3G3v}I4BV{!H%qbvK zbBX7KJj$Jj%!gMhZlOBwOpBSRx`Wm8anQ*J#55|Dt&PH|D%ln6Hc@`!r5vvbrL3qX z@qWC^4mXb^dYwjGVWM$}KScq(5&PbR>m$Vhr4GEmn;ZYK%GAsJ zyjoB&S=nFJOhPg|Qyw|?edJ`!{;AioTlaF|1XDsdGoeJ)y9avsa+Rc*Bli>Gh_M;7 zw1pspUH{v|-;|?ys&J)IGF9Orq%wCH`Nlnkc8dII{`{9l3nS>5`H6_eOb4U?I2DAq z0nh;qR3VPr-TNJKimaRT`bxF@Tc*RDpsndym1pZ=UE}ZHb%efwT`?{ey-^zWN>6nt z2kbM{9vft~!nwJfkQPlo&nSJfrtPLup?;4)=_^Kwcf1=PpIg@R98FZ@2e;pmQ-BuI zKhS$0zdtC>(KnlhSV+T1w5xfKtn)0`RFd7?JbXynoI^o=aL`_Fv5xzMXRQxI2ww`_ znUVY`ub59QM`gOYqgKwWQprusVHrjiTW%x&)?_Z90E9B$z!OheaCMi0d@vQLTRzSw z3*}N2e;mz7CnN;3vqmscA1VJQd{1OIa7;Yt7K{$quj#XGjEGKhyP0Wu6WmnsUVUbt zIQ(xMFGi?dPb8|Bwb-kIAjWbhL~|=fW~UA|vI@74#*(=)qvy#i3nsw~o?Q6pVv1E) z#m1%&l?f@g?+@Z-m6R$ljB9ft@_r zr4Vtg^E1dK+9}?S%=W<q?~;vj6{a_72<-ag z2*P(LloyC7ron~`i@;SwF@y2ziT~-n&ubGZlTGoIC+ya;HDYCi7QsWf7+oiVrpbVx z)ZuHbKxg`?h(j>L_{8Hg{F1|7ShPoEu#cQR%Pk`tr>?LjgPbjM*Sn7nD{t-;yQk0m3O2rUDalJ4M&({E{6*@mUUxDMn+NpI+4dd$NB#6*P)f7naK%wGGqMfu<4rnDiT%vn5sh%G%JFE=;{_}tt|so z5EFI+31R^~fmhTH(m%J#G%4(?|1#j!9aC+xw6pa_EzsI5(8Amg$3@cBEyjIqmGc}u zK%#LVRtv|``kN(k zPDD|hvU5+v#heXV)X;?3j(dbTZOw^ClaGdF8ei{aK?a$dwMS!m;|ddwbY~0olNZ7u z!JxqMRW-ByhGR;m__d(sCENkhsG7oN;{A#X*sxKtF@P00^P$3mAF*Hvvp&h@j6`+P1?zx_i& z?G#uU{UX!wg~K9-G>k_Rsfi4z+t3(XJP;^GrJqxBIhWh%(_rDJAW;A?F&up9Y96nwei$(vllzCpZ9Buf%+5SXECggJ;$tN117>XuryI9o8 z$*@JIG5>Qz+-at8ne%PZK@yGu{7J^w^!BLg%tGy8LPiVKD)fHZac}YfVk2ELjk_BV zdCBI1TZz&;;=dyYIiywl0P_!crQ!D(5_y5Cyv(6LKVD^tOHGwR2&R)wm1_ENSz?89 zx?`rf;doWZs#2ONxU%kdpR~%B&^FaU8hSPUfy)e@Tn)o}B7S;>xNcewRK;L+*jTm8 zGM1&aS?6USqpp^|$Xl6*to>^_=f8%}m}WT#H@&C{5WP)`hOO%U+e;k7NgmqT9*duQ zla;CeGBb^n)MG&g7sJ>5L%RQJYVUIAv1iR^LrC5`S2ODuilDGUTsBVBF*OiumgWEx7?LTR`^<+h1gF4S!yB^7Md^;5$?md~SLTdc_-|-S@{=)8 z9sj%GEXk(+hTKjVE+{9$`<%!w1tDTE65kB@-c!wYgD_OUj#m!`i%BT*f|!lh z*lo;?fi|!Kml_0Y_YQJ$-G9dX_M_^)w_7zc`>ZSV#x8)TTJC8*K}qrwDY;-m)em7zVz@kNw>cQhYZBzjq8 z8wp_!c#y>pS={$Yz)m|A`MIMn;{DB-t4>YnwKc9-?{kyCy7LsSvHK1F{EhbM&HQ1V z6ym6HhE~bgVZ_RnT7pU~Lh5E6waoEQu;bG7G+i5h7M`+NQuBF~Zz5LGb0V+eVw4GgFGDZxD4$KiBwg&oBc zgZm#iz3~o(XGKY~36t<5CUa2#e(~u4X5!!sGXC=WuWJgz#w$YL#w&+ZJYqV|?p_$d z)0}GA5qcAq%$o=v_uLmH&1ndM_8ij_69NF_uIqe6LaeJFcnxC;=?FtH)N`XJR!QV6 z5sE0wKJn?AitHd^`_Qp&@v?<4nk7R9&vHgA_imqT?wbiZcG@=ZAJ~r7eJW(+?^x+R zqkq-{_IZGXV^DH($9eK4zf-@|R#Dl)H*|@g??_D9PzGknh{#QZkb9|74`eH)-fP4#Zcx-$sL<6+6M8tGDZ5 z0q-@!CW6kr$+fHIy)3WG)X#tuzq^mo^SR0A4ea=^FqO^_Km7QzYF+)$`Tk4SwGqz8 z`qamkEH%d$dmB6b_iIGI8yvqGdI2e8`YYHiaL*Fxd zh4!DOIGs5Ghix?G?1fr_TKd|l6=ihv)pTpB)UqFm26G1q@l>3}U~;>xjEW)%dU=M- zhO274t5uq)H1DfXDvu-tyflVH%CiVql&sp87Dq7Xsi#rp_}_Y4BP2G#`iETxBInG4 z;ZAYh-B>3yqc`?yOMGr2LoFT)DmX9zL+U(`h!9#s7Y*;Wx3Q z|1fR=yweQYZ;H~1TtA?J#eLJC+s~C8Bg(oCpGo7K1l=px}wkN zTy1lLah>B4Vemkr+#CN0i0kVCu9*2ke-LTNWD#3RWl@O{>Uqr|%}`a5rktzPq4z2Y zvsIEPF>jQtEhT1{(!2sLi&4qkvofjjy@IzcAxmX|RMxzOHu={qaz=~tgaP`AL%#5X z!FjXx3204S{>E)eiNa0guea+5Kz6*&!336;UOOi+R56gd^|JBat%&!YL8RpZvDy!( z5=o_26|phCvVbeuVAYdqR>YXJ6uG%U_Rc>|OiVoU%;wvSK`;D5_SX>*_V);aFUM|qzx=)i7__P&Fr z_SQFk>#CT_9OX5P!(C4BLv)oe)z(=HPbdDgs`Wfy6CtIieghXV^b*PV5@cETFzf?B zWUj!-p0UGlp5S?`G3eVf_~E{q!cUHu{fyKQHe-t6)RtA{|ubt=`OdwcRtW>TE zE_nS0^y%Pspo2e03#-hot}tLQ>+bdunG!``_1P4U3UuKXjnmwxLD;Ydrn>|Pr^KaV zU@lj&7P=#?dSDh4tBiG!!&6{4rfxS(z7?x=_uL{b-_76l2#9!_L6d;g5rCW+C|kmo z&Y;7t9hU7W8}|239Er zpf(V4q~Q^bq*#0dSvUhPgPLRRmH5boD+{}27|u47M$d9Vo3zTnVBT^H$mSZE2z_SA zPWf)=>9;k!3bWt=ClFAQD#ed)wWkSaBWK^Owp8KIC`c9+$XcGp>nzl}Gj!c(j5S+O z+5&!;T=Qd*6xufQVJH09#rNQ(HTpWxyk3#Nhe%X(#BvoF%m>;9KL_gAb<%__n+1Aj zj1obwh~%|OlP+n{I zjCA~!mS476;~y=`HOR@t!6VrzM&kP(@#0Hz*(4^f-p%1Ghy-3DKLv7jQc6PF_%i=# z*$i~l!Bc#hN&{sLu3=Tc{+=COXfth=-xrk!w4VU}%bRR=|Gd-0py*6~F%OW@Ua0?s^?>@i?Zh|Fl zC&rd~Knc&~POv*tTI!W-u+$q*>44vPTthvnE~#0Y=7J^wPRArP=FX{G7Bjx+&*bVUP|%}~_U{* znr7YgT&4kHP&4Co?s>=#G;H}pBP?-qQ6Mhm+3r{%6;^|Z=RX|L^|kide0(U0{FEdU zR=LmyV2%=UACYkCQ)g+#&e5$6dS%OuWU9{abJXo2Uf9TZDv0yvii|R)?+PbhO;$W@ z-mB@(G<^FXia z+;&x)A>_|9?yf8+f&B@`tntXdy6t3sZB0h>k&FpzFXdAU3ZTr})uWzdgs<6O+{FBD zK%GEdD-j4s+}-%8Q@2vB$+L+Ct@)R1^kH*}NWo{8?GH)Y=_JM{oo*cP7p%6T|4Q~8 zr;j|giuw})&s*=1;8pNc>$SANcFWtOz!RFH*KwhS->JQU$JGu#Rmb7IHLZzuTHQ?p z;}Jb+zMuOPuOPr@dd}VgfnQ=r(8qW;A91ol_f#DY&dyNgMI*h_udf2jp_nHJ4Zbq) zJQ%ShNlniG>nBR+e=2Sk{DogX_|WoV13#+ZEzW9mTs+g{vqZYb8@! zu{}|$+l}t0oXP}5vb;N!NK6XfZgZUdF8`WX?L8iDw4_0HvJf!3M=}jPqTg{(-vL%S zc-O_KOGh%WVNMfRPul=c>-IhIw-|q)|2wB}qBrs6P01S5X6CKAt!D%Yc0Ij)JT!Ta zHPJKDkF*sDKfllP4*+ha&G)sg7kvEIY`ID^2uFpWx*pRmQZm#g6?7akKcPmb-fv%@ zP*CI&%nLdVs;&6LI!M*@3(|mfreA4s5B3kK$Y*S7h-sdKB9PNgF_PNP7a}SBA4UcH!6Za_S-wY2d zUj7zpVlA6*t9_-4skO+rktkgFRN}}Z4cd~ObZN=0PAyF*3$#5qsPqH?o?>Piahe!*PB3IjNbjVq~LXTtd3o)FGF$Zp~wzlo7(0PyBl96 zdPJsNBrJ=7u}RM&R>YH4s-b){Q(CV>S2l*oQx*N$e8)b1*G$p&F>ChAIQx=521^`6 zMzMRwuom9XPGUqVDQStu$OKM2{+(vao)Ju{?DGm{o83-^uGA*5h(vVJj=(B?vqwIE zy7#n9%}Y;9Pc8g^S^yEEB4+tdT_prlpBtGswB-n_WulI8%f-{tDiMmA^9XhExP~X( zCF*j;)X`E}ZfZe!dhh?H`EB9DZsLPvW?lPPE$7NusFhK?H7x#58*W?Q^*`!3J9i!g z@B0K_Bg(UK1G8tT1U_zn?P0TQ@cU7YSLa3dANX5;8jNq+Q>gwS;PQ_eWz~$d^ZU8- zR}^t6gmBS3B2g`3rH!;j4mHs~9MdQAc2xyyPihrzJ+(eX^$N!Srsk8U+(jt(i_ss& znX^YD5uY^``HOpN@KBq@%bIUkFNhFY;5m&W^*y!CzIx7LfYOPdKY)O%gzweOw}98* zXlgYrL>t&1t7_A~ldSgbGXQ2Zp?hidaI3np5!E#bd7&Ui{?X=WA>#Q=VaMgPWUh4d zNE&ao-qY#IA}+f92?#rur9;UHI`2ghHk7Thx(NmUz~FV0IU~BoKVx+L&0-?e@S92J zjb=V?U-P-KmLgYF+s6%&v5Y`CDE_vOF#uYqyr`s>e zU#`8KShTs5XBY*BLv$KM(HP-}N8f>paXwFLHLE zVO@f5vR|q#2zbuE_&o*_TMVgFHdst;+P^1V0jz-Jiv~JieooC6{B?{(w8Q&x z7~$a|KCfzj3(bVYj1mcY)2 zq;oytOG?N%`X6|(C85qos+-X(odga0o&`&>Ed6#`WPsr90d-!@mQwq#Dq;n*bF zlRrNshb>4?$A%wsatad@;BF5bROI8eCXRQUAj?{SX|yZvO{Hcmt8ECjKuPA71J${) zObzL%;ZC1NC%F5g?W&bEaqZ3aNCc0s5OD0heIw$esri(Y_es9$-je#(`sQ@>wlG=W za}mH8Iu2@CWH}DzvQBC0>9>F$*X#is{+b;i%K{%jn#9l^^L&X4Ci=qaQkqqXT@EYZ zUj$JZ>b8uSpG{PGxIb2D#12HVjXLlx1HykW2*9w}(S* z{GO#7$&;c%WKX=n$wSN8fDz86GsY7PITZkVDgpa=)%DitCEBL(_1$e?T_>mA7MFI1 zyW{rzlGW-*-HA>yl{}?i{6S(NA`ys=diGm)GWe#N%D)&^%>kK`OZt(w^UOf-RyldY z=P5J$`59ItSpy(>?q$E;AIULvyfA`)^@3mN{XT{z1s~7rr+gOcyAH25AC@pi>a#%a zBUYmTyA6EV<@dIKR4j0lD{7H-_dHgevaRTS{s#d4oO4bq_>iRtewmPwJM(&~w5t$! zDbMogdsrgu+*_2?y(K))6kJ)w~$$N?_5huedAO&a)^U}&UN;4?LdpU zUDWrfR%<>TM7N3BstD9XX;A-iIYw<__R?O`&Kz=;(@bmC*d#jJ?!=A8&+}nkAyTFGD=C@Q0~wkW3~y#=#Rv*yF*n zSmkQS2D#LA*uOE2tNM<5SY!1+E5^vOaY`P z+pd_Sc!%vRD~zi$ue5mAi}u~rH6_L|T&Z|F+VO2YlTG zPy`D{s{cd9-5ESj#G7?R``!E0$YYxK-QjB2gu>Oi!(3VZJvNXyr|+0Ak(ohug|FEZF=T@bp0l){XzO7u@>o_<`leQ)Ri`y zkT@>RGXFF@-9EcU-55? ze-u8O%ygvv1-W+kVi)3cCkuWfd0{Aw&>P|^E=pC2bW$4qLWxTem5qtHywc)Lw%il| z9Is%sz@dJ&`!90f1I}ecK&*e^`rG;q-3qgIraVk|bLLxg7gXpf#!A0u1Ac6aGNdQ>2F>HJkLYaGZH{@+D zz8&P|@AxJVNB@?Ub^43TD$j!jb**PpN91iC5u0MPL>l8og-6@U>}|T22lIv?TA}j& zTOR;*4J!KbGZ*oCB|4yF`Vq&b0c$j>{YVBw?t&_61|=xO5qnVY-wArtuTX@DU3*UPVvj&5QvLLrozjF5Ex8fx1a91MkP7txQ z1^TIVV7Tj0TE~7am6aeupK=q4qj(KjDEI- z(_0T+(dXd!ITA%>stgY=oBJ6D(@t#dOBH`y){8*R3%-ukL#0ybcX0g^@pclyQYwl2 z?ut`20YVjoTJClSEW9jN0L(XWnLIR#*o6Kh?W%?R4^KB&x5WwXHr}nv@n6yNNmd@p zc}N`?Um)U0;Lu6nfN453rc#NB$K7~`1uN@1e0r?MUFTC7tuJ53>($8cFh*?Zj$%+M z=Z~)7r#{qvZQUUqNU=RdXA(V{fyj!AR;ym6&Lydre0vUmVK>au{*a5!vPoIAaSqpo za-lw@pn|EwGVEgL?&%QagFJ1Qf^VbCf#2}@nx{E_kJ8->MyrHrk(gV=)5|jjQ+2{D z7OJ?<$FP0toIrEd!oW!1-&dNbmY1P&`$o~?dnq76Na;?*lJ^~1*~fGEWeKG_(kZLZ zL0QMYJ}=<3Xa3Ljy)@ie&u-s?lJLK6Ejl#u5Bhf>`;G?Jm9*hdaR|C(P4W@|?8Yb0 z%);Y2JA#kkbaP<8_UkoO#_uCA8OO=TC1h_6{9ZUW$@5g<%s~B|G{Fcu4t5h7+==8s z84oag?f%%j#d3D8mtp1P<>z5RK!hV`H5KMp<#0g3pqI5|&!YJRbqzR*g@x0DUXtDx*v*@&B;Y2_9->c4@D5J22 z(X-0pfHU~J%coU44(N#MJakg_$c2)nPRa%-NsMZqKc^)d{s?T_Gd|`Ct9N1)pMC`V ztgF3D7(g(WsjxicNOAJT7#p@raMZYy9S^Sz4l9tJ6(xjo7PJ&5d_2E5 ziQ7RDz*#a&Dq7M$8g z>2ZF9MWDc!s6FzRO2)w=P22t>dlArdSxXQzv~%#GIrJGk0c+vpZum{uXurIKPy z=tzfJWg%QiwcP5&oj~ip5j^q^o)W*Zgl{6O%YucZLN?8(stQaShaM-B?2nyxVolU+yex+;O_435Zrx$V8Md~7~I|6-Q9viaCZyt z?l8FBdA_yYJO5{$nYH`$?yjz?{zckNH>>0*JwzT@);l8d0x|T_Og?K(>ALOcUm(n? zDB&w=YMcaF?l#~+Gxygvf=kYnbq~e&Fi&b$O`H>s^7r39iqrsnd@8d_(WZj?F=FYA zZ%#qQ*13T`Jq)NBfxQB${?g3ZUvHkOU?`s`m1{1jVpn;jj0O)wawB*Htn`@9zM-UM zR#1yFus}wo(Bpkm+A%hjt)EgD2wg7+XhVB5i#fZz5;WB0Pns}JixLqlZkUCPHSe+-*=-i zNv6Z5IA{;PDlbcrQc60=MF>y9*9(d@oEpscNaAk!pdtd%ZRiZ_lQK+{a+-F4Wl4X> z7K;G+Jj-XXkB5hUgkSr=9QD{-jHUvo|9#vlw+MT>`PO6s=|0Ci(tI>c9>$4OmKrrM ze_yV%2?%fq3*X3mo_#tm*L;id&tTF2^HyPFze8fubCa@`;i#?WVvsr)5ZyY} z+|vhD(A3!2dLae5+i4;u>=z{vCQHKfk#{2}fPbVdkUr_!^A#-WZ{;Dm<_njh1I}w8 z8%v8c9-eW^=xPYBMT^_IN978NahEHgfy&h)4v@V>HI)2B+1%=u+IGfYkexwKC9NN{ zQy;Vm&}OwsQT=GU-&jA{VrPBDuy^c{|5*{vr{)K*$@<_Baz?EjUkiV@`6!iMlkFMJ2^L=mOaGw|FS>UGcox+lN(K;Evgv zYOEb`1j*H;r-yVt&i>iPY$`}vUGvxXD&!Fiqi4Q;oh(ixnsJfkXlI7xeJesq8$fM< zOmrU~aLc3%yhf#RDY{ZAIe=pkQuxFu>05BNnuz8*VWm-CRT0ua-*;*on=Z=2Kbf4= zO7q;U`COs^@TjF_3O)Tp`-nsUF9)eO!D_%ll><%XE~B3Jbl)!qREeBmxri!zZo!NE zF||Ie5$L%e*h8x}Xgl3#c`xAUyQxPU89BykHF9YXNv)Vd@uiM<5m9ch``;t77At;1tJD=&bH9U~T zJht$9!i|HzwqTi?SpyQgM-WufsTwiTHyNZ*Tq&|8#f$D}NvY!}tThC+rIVxY7x_Kd zt6fl{aDKUZJZq)K$hJuJ!SUc_>KN<)=p!Ok2y-d55a=#@#Ib3r~v z001gxEI$)kGbp{S^GV}MOe8ZqOR@^;{&y^nNZo9uWySd$V?9E}(%?E>Dr8fzCD-4P zx_z!7b{bjHJCc)1a@N#fzc4|mBh3sdk%B-&HSJk z(6_Osbz{-i9e{ZMk2nc}P+Fdh&!+BeFJeKBCkV^u^4j=K0|!(fMWJ(}?a`m3o;!f= zN7CJ<$J58T~FOXa_aA^6Qb1-cfKv-dw766uEDa>O6_FQxsB3;?w)$d&6-W@id}aYM84{OHJfX%#?Hs ze(BHuT=!e0n-h`}1b%ln5&>GMAO8fE(tHEQCYmh##Nw-OKBmX{!ry#@33Ib1_@AwW zKJ-nV5Tx%K2#-8Uq0H%%9=f3>Q*}Nx$rUWpdRiaccq+Nri$_qLJp{Gy8VkRq*Rp6u zU{qi&cLXRtzl!O|Z-U;{9uvZHBQmS$-9={1;2IHal6U@`bOWr;&;^5Ao4O?Uh~GBiPw;}w$e(~{h^bC2`ZiG1F5f$)xg?P12}dOh*Qg=ZsA1b% z1?c1)evzm*mfj7xQRy;FwRb~Cn?+Dvw9>_|*CxFy={bctIiqF@I%J`Fs=v^?$!{eU zyjTn_AY>Azn{%ktkPLsge-_L1+>c>yC3}+osO4GCJ)54I1RWk)SR$`f{#Gmh4d^A- zeVD-$dD?7=6}Txf9ov8Wocy#KA7?oiaj#1Q=r&v5*BA<1wpFfs_&ncYw|H;=fPFq( z76uq}rnx-#K6bmFc?yF*(B9{K|MNQqQU0>8lIyN5<1t@BupFplzSSbeeyU~G(s|pB-sH>GQNc&Nm?Q=2_$Yl{!Bo!SJ z;@_b1pkP4r%@}BjmGH|23Tc#v!Mq1H8!1(oSAI`p7)j%YX6rXdLaQSZ=s&J-?bczQ zlouu)>l7O;bSIMuT4rWMmyU-T5LL1HGbvpvaeXt&KX8M{AfkN<-b9b`0E=Lz;K5H$ zLEzNys9(&Gzlj*}N;Efr1Mc?=*eLp;eFX$Zq|Sn!_<*~dvKu8f9XR^gNnON~nR0%s zzf<{opvF9n9=?hywdpw1i)!g2eqk}LTz97Dn7-JtBtxUUODH}lKF_zZmt(u=>j<`| z;QHs4(*M&e17d<5XOgT%#d{xX+=V;%nL+h=fsQ$l)k!s<2bFM#$}I!q&*J4BL)+82 zs4*1MeD1C=8}l2#s;~y4bj)C@hw|JLxuPFQB)#rkywC(O+RbslAXgyje3X8W+I|rS zf!uvU;R}Yg&ec)VcbvJ%_JmC7=i`BWBs^-JbQi7Y-?spB;iN{2MYq2%T}FI1?Aj*3 z|Ge~ol+!CM)DMA;gHiHd%UBN8Ug)$tPunw77>QpPv5pqrHR1KcAL8AI|Axe2)8bc~ADEm4!cM47WbH(LPU8Qm2JJcYnn2)uh>Y zyp+qd-6m`2h+MYbk84-yv{_G9>Kl0-9N>~t?8y%7@9$55$W8%J356$a1RhbodeN)SM%>eWh`A0aN8JP@a~)~)!#7bqoxhdgdQus8C3sn^|6u8ZaN*3ME2<@rA7A=TqHgJ zH&qGI9nufMm&;)8m}k?<4&gssWVT_4oKt$6kZ>bigYkaAy8XUL`C4MQ!-a~eXF-3e z1&kWucW#&Qc_!Pa(eP1;`B2gu>3W!VQ3LHbC#OP;acyF>ekonW#`yU7+t2v;+4|Yp z_|u+xHO)8LkDWV4ud^4YEv$XO3;j>Iv2@&e7Vk;{6O)zOgT%z|ZWmtPFE_Ii5cJu= z9}W&^aQ#7BQd078xq@kMmUJ0w`kFK##7BO%BIGo=+C1+}ffD8u1lz~a>{W5ARj*|c z4O0zO>r)gQiqwNss|w<8?ba=+7bVoAf2($&K$-j3`XhW+ZT76OV%?Z*3kz<51okz> z0ZFHkE-!^`7HNZHPkbpd-z5A`8;Sfky-%PMRRUo?X>V3k4ku$??Pxl=K&$n4CgFGiEUOR=AMLpBxnSPs4F2;zf;UjQ-xd#G*9etU83wWGX9lrelRdm3#BzcM_{lNJ(G0mu!US zeLW-+iETi4MZBnZ#4j?o8VcS3t@Jjfk9?pDlo(|k0h1c?!a{k~i;PQf1&*U5cd$0E zc@4e&bctVuze;9ERvJO_W*pr91&9XyZ0?-+dpUbuJn+d_7~jFe#h9V8k<<)RbXzF8 zr_Ct_b0BkMB(rRj{k8;r0hx&2RVSjNN5Mhl0lv?ooQq~8peuD?>!o6+0ZBgy6E*%u z-a9fGqY&j!t|MaYg_fZg-j~}bv!VnH!`#Wx-jP3=`ejtgVSxkxo9r;e#n;~(0GH*2hL#P)`AT* z-4CeruRN8U0Rx>kqo;qQiqn&PjEP~rqJN;z;#ss$&;=}(6j{MNsDtR%J59;IZrVyF z#29Bm_^%esHsZJ8(l+q^%e^+_o`sQR@1>v!33BH^nFh&1%{lRIyrGk*Gz_7sDsrQw z3DHeU2nuAoG5%!}%cW{1SUu#iM{T^HSVC4ZSD7{tq5xXRa<(>0+PiPhlOMpy)rvp= ztZhqQX)`t5ojE^uxzSE&OPA*x?=M?PqL=uJgEsR#KEex??=3IdJOvA1EtA(^t(z(` z!-GIF=SdlPe1eqQZB<_W+qcKM*CcnEJMwxFaS4}qculKZ zjG`d$r-PO(eM>FoBS2Cn^?K$7SknZ)YrM2 zD_EStUnA4bpmG&^0j@HvOlG`uILDj~QBnQa4MHrdgzb&JvNFZiAz|cV&MTdZL;;$fUWv=pD1@sWDXe{A@xdv^O4A;XK9~GC zf*1^=ru$DeBkoRV9ah4;cB;J$a3_^ix`13G{K__(Skv{?7*?=RR~gA=0kjb?;YpVo&Kt{&LF;AY%A(A~@JIjo>@|5zjiRcGDb!MAWAXH41+_bc3y3U&G z+iRSa8Gg?Ud8^(PL8mQMJg8xqcl1_OH5k5CqK;iFM~iEZYn2`CTBlAxv`a)rX7c!e z%uCcdO4XrZUB$*&^2RO?#nrH>)BrymK7J^LEg(_;XlG(~bMcrBHQL>^VQG9*oD2Hz z!Xo6tayxxY(j(WA=j@|@RSd345mElx0og$Bu2wp#Q4N%*^Oj)9EFo3-1VbJN(M#De z6Z#ewmYr$g-2yJI+3lJBu}w!AYthozhAW-e1>xKSLBWuDoy6M#jyb3=Eq}tBwCieq zu0;QeChU*W!P)s}$>ejWd90S?>p$wI0KcH5Ps^MY7%NMGTz6Aj2*diYf?pJl!d|(H zdF1PF*g*6IR$UGv0cEC>2vsCpjdUgvC62Xrmgyq=fG`zQRGlG{0%i(|epq~Xwl@v; z$ie*zLnb81*&!{W072CTF+LVz)hj`gYxQ!ahE>Cm;X2zMj|X+tcY%cqcT?okbQG7V zz2)R?#@`|3x~PSw(AZ0vwe_gs_8gJ=nqZD`{)EXoCTwoB3S0*Bsz%d!dWgd_bT(7O>>>@Db9tVJCsdfU|-E6%(YR&(-lu6iBbE zqbVFJIyuoImMJStykQZ?L*sh+AWG~ZP4Z4rJbnu%b4d)eVLmp_&-Kshtrm!dkO34K!4G9+Lsd7jX z+^xe_t0Ajz3tU+q0qo-%86p9yuPmVty>Xh&R^F!4L#i;lNo3wLy*usU{c9Y%3khq{ zTfZ^j4>Wy)CNhHFG%6CSHqrkHGuZ2+T79q8VDehYiG!!D~(vin@Ixa|FyR*Rr*yLu7E2ghw0Y zbBFM*GxY9E&a_h=(?i|_xk4UZ?_m93EFhETlR3C8)jj{k_!2(4o@9$YWh&S!L3P0O zBMzzAs^eBTo~AZ7z^@2NDiug;_{mZy78bpm%QWp$)Cwl9#RC?STPSmTAOihaY!&*`KOr z;j{Lv8O^IauvXO}N?$Bn4!y(W#^|ahXFyi-_Fw++xy5xDtu;vk`s=<^a=Y}dg_(D= zlNmtyb%`KBY>)&W7^<=axqPbFpe?V*A@y<*)@SDV`m4OP{xGrl!_DZXlfVI^RB;Zq|J*O;o+|+ z!%n1QZ+8(&ziAL$lon)scsYfm{QE8f=c8CMv~s_)XwrZXIh#^4reS!PDI@1o$U_tL z(D;792JvqGgM!@&d3GK}hLf;2&r2(6cY!^l52P6XH68Fe>?nzFxTu)d*8OF|jBPcH zrx3QMK8GPF8EmulSkTf+)w|7PgoXnY?ZY^wLk}De#*Wdk~sHhrME%GG- zDx94|B}B%N-fg_uW!d`!w*4$Q#oR}dF;Rt=jHE9U!Z$2rz}~?G${}r5$e&Gxb&5wQ zfnuuQ`bDUs+9Nj_Q-btyj?jfQ%Pu0fTJD497yY7oC0h z-I~;DydNJQpCJ3PRNTYeuhLnL;A+W--q`0AdBB=uw-h7AqGO7-EscIYTchAuoRs21 zrrqHCA1Bo|ABY*|=qL616A>RpE|s7Lr)}f`dU7*YX1ArdDQan4RFs*>@6fXuimC$T zGiy~-51RHU>3f-dW;J9&nvO+s)S>B^9C21@F;QeYTDjx<$Mj53Hs;c%KPYXt@oA?0 zlmmbb8b!j0aBcvOqlQgf3X8Co*Tn(a%;i9;yAs4>K{i9mRw*a4=N$31^KeW3FqYOJ zv;QW`vuKPbxQ9YE*-S!rju&oyVL5F;Y0imR!i2NX_RxsehWd60d^^?56@2_J`C2rP zcfXoImr978d6AA$mnABNxMzc7mL!AX-%12E7Nq?ZBIKtZA84lu*}N4En4LtsWNC4Y z!$%fkU%WtbWz1M#zl;|JcY6P9hI<)MX)~yjCvLh?{8g(Y(<^(ZTP|kK@Uo!g^~-o4 zmy*PWpyEU>{*Y{m>pibn)fH#Ku^q@dmA)^##02*p0(23Z;3hXZvMt*3u{}sCaagGf zdJ=9d*vW(bk|!Z$#D(*R5%QjjT{^~JH~dNIKr@j>n1F*{68 zog#{b*zWKDEofwfmtSni;PgQqQa6EBt^EZqeKww~i%K^@cuY!Sog=ou8fGUSD&Gii zLjrcr;Pe(vb^<5$SfD;*#S$QSRkW)i z>z|Mu{+k)v=7xaaC(R-P#qtk+CmzDFLI1qcHc|b^h;hqg$#71%>PIM7uLpmP8uqVW zFq=2M;KZi`?V7H*8kvzV06Af*=v-rm_g3+V->CCcP1|K13-`H`ZHOlBQnYlVd< z(=sH5P)kTC7dvGspfRV6sAGsXSVL;V5ulra`vz6*w0F8EVkZJODsu4rzW5T*+;8I} ztR#rd{Gf&LI=%Fw|aY7oBnb^;dJxLA%8f%4jiV|r%u7=#Q9^`};5%+|c zALpr6Cy_GbyT?u7zijKBSY_`!JnF%)) zZ+8r&o#57-Q}ca!JqHE*LRg^vIKY+}9}g{_*a+d*^U{whv$`zX8Zg81wg7S2WVycnh?^@#N zafE1Y>@nXPf;;UAoK-|<8Z{S?+rF< z0dBQtm-Kge4QKlB>6gw8TMcnPo)On?S9kJ|c^9_Sr)1?a6!v-PL0JVJ3n&!R_b5Tn z!;f>fNSLTaNBq>VcCNU4G*SdWJo`LXv?ZmNH`O4QcU5b7xMdRa&CLvO`vC)-Wc3wD z^CWu8YVEOBW5NE>Jz`hqza^%C5^#h5s}}1+Ir~S1{*-_e%GmaQeGKe^f&!F#M+gLr{=}ALc}2b>50q@81J}r!wMHiH*2Up9%6nUP)fOtTo-7 zD!Ama(zy82_+!=2eh}m{h&ds&wE=SW&!Em>aPI@>NNm=q`OlAYbh6pbx@>dODOg`t z>joNp+KU>*)wdsjz%2^fqK9uooG090jN>f-<2IOnEi>Z#9~O}G`v1@DLI~U{;X_^r z=yp?nq)NRy!2qa}grjC3zrcS~hVxoe%_Rkihke=*sYBRUsJm!7EQ&}g$uy{EoutWB zaZ-RJ=R`BL3YWftIZ}97c$>p>GzW*13-lu7y(~KXSFZ>VqfGq&`DEjg_~&k>e_-#s2g+xE+|p^^szGjJ*?nK z_CWv<*4t+$vy|Vr>6-g>d?>~Ffm#$~VrUuxg>3hvPXV4~lzl)Nd^1KRN}L>-xi7Z* z07jpG!c^r%Waj`VIwD3?G3Lfx#Qd?1Q<~U+?*CLiD5i`xqW432Sw)oT=}iK;+q#%< zfjn3mKx*Ic&y6Jx0_~(tVGBH!deQ5ab{O|S*U$RejK?pDn?rwU8J$}bjvYZ<_;6B@ zLq&h0r1%nvgu3arSIFozpdbpa(nm%{uKgF=D*PB#2XD8GizT-N;QFh&W?ovTtvU^s z?wOI7abF^o>;gYV-ZXo3SK1eKH*6ya^Bmgm{-sEJ_f zI?eE~O;IF#8GEMfJ|>R4L}lL<%IeNV+WT%%A^Vht#16=?b-=Fd5ck%qAnhxL5=D)b zWBv?gnLc4;|#CnD-?Vi@E46?}Pkar5rZH7eAEH(XS`! zIlyu{fpUuUI1q{q3h)i5y!$ zmZN9?NwUwp3r@04+!rWZWJ2MskjbS&{K*HuUt-7XOTRRGuMwM38Tc7bdxAJlQCzJV ztAs+=BDlUrep1BH?ZTH#l#KAEdyBqDrp(1e^GU;fh6DcM=6#5AcIc6&j)03SF#^#L zhlpO+#_*PSdV9`#_@sZ6d0LBWP;O}0(!8R5KJ;c>nZ{sVBYFk&-bH)apZY8$bYTY3 zp?QD#)1EodG#Ud8*pkBGudD$*{6xXlWuO1O31rhef+;tw+^4pD&i!d`!OBXh?sp6I ztSgZ~rv(2rNk5(F4UFgBs0M!b(YYB)^W}`@y(G3> z{r94g%-RnhA-KQbagAcJpDIanmSXvfoPCi>s&})i;YPX+E^0xkr?0u0SyQmJl)evr zQLjcaPUllG5qcErFnQdnGOXmVc+-mBc_|cStovhle8%{gFm}Wh9ChH#Qp2q-e!L+m zt6uGd-SnRTecpx{$bx8`@(KO=xGC(X4kPQ7!GEvW-o#VIZBlOv+z4wus;g4*w%Q-PA$0Ku zmAeJYA>h_br6(8ZpJ?GUHvS)${+lty=PJ^4kz;MhSIy*^W!(^q)QQ7U-&J%h->^;Q zsG8h9bDDek_|Jf;J^Eq`M^=7M!LFh!(h^}j+UT$_o*77&oa0P+zIj2;Oz}o?+Sw6g z2EeKxmGjG&CM-k~#FCR!a)h5Dq7)Js(N13!cJ8eb`lb8e6wC@?re&m`df^VMk!@@T z>m&7D{*A6?L0&Oa*T8@$nnOL3PO>usL?}LH?qx;&KoNX1Gcy>~ZWcMu^tj(?vy|Vr zAMT-yd-_O3Esfa^!;+A0Ha(0*=WoMz-S1~!ITk|bLZQ4-pI!(t$;dqSK6cy)49fkCbt)EsgAwdxCaH^M|A5uQ zIL3*G0GcT}TWDqGsI4>NLPwuCp4j(; zREC&S8g&f~f0&l1oOFS7QF{}HGYhh(77@-%1^w-V3h$cb>$G_^v8@o^o9w-c z98Ab3J3BiO3=h)fEv1c*+qyRPADq}PKkAw~4!Q^=x0OX+NSH$n+b zR7n<)bmJYAXNRz3Em(7f@y~!AVjLb;l&KK(t81j;=7Uy{5*$1(Fd*DoJxgFE%+$4H z5(U^6w3oiS;LjY6?FP!mS7M`n8bt`ZYT!NBeM3 z<~c|nM)m^?!2wZDgp47ZU;V*S@UB6IJ;`x>Vojq(%|a}55&zjnkXML;Awa&;VlJU+ zbq_=Q!gF>^|YO-+vA;t-f$G%P|HDS|v^t8_J%Esw047m+NP4+v>k-+&Z?@saRkjq^8Yn2Mip6DdP3&SB)=6#5Tz#Md0`FmgF+GLXl9{0U zpQx;RH{#RLG785_*ZRRb_R{1tPc+X#IMc3)gjcB#3~X_#(UhDVjRMBF;A=~QXw~As zBb9k^#+b9@YYp1>R@{Bf_Pemzoh1~cu0G;bE-QkV|L^cr@^TA%e;LR4#Q4|)%Rq=S zvvG{_gWdcUYsU%-CY84tQ^9^#1LFTY`K81a8+QWkSh}`%8Km$!-bgEJDz1-=97m;a z0llIRNE)mPnesa(frCk{5#QD^>spPrkODbb+uWwQDaOKl@~E&F#cjs4+O4KEJ5<9l4EGwGklstpZLf!{Y2Y$wpDHvby`L0y0}L^bbolH#(_ z=T!Wxsw^D8Z}qQRx>6(0cP5HL3h9V*53FlVcZ=e~e__2VSZq}Tu{jMp3ujw5N<)bb zNl*UP?B?9Cc3mQZA*iBHqTUP8q^X-=svnf12==gLaO|uG5hv~sp6w&_{c5TZuGK$k zh1!R}Hx2Nhw+K{IE!HECySlhZUxah57bh`Xb0|F~X^`xX4uJ93{EmAMX?k8Agw zsJ!W7s2Z94kmOxlUcN}kony3AaViJxVqbb5RL&JZ%HLaniFr{UkvaBv3z2Q!YGHub zNd4*wRefu2;D|p9PZ{cO_W>EPFCT?8SHg^eIQt_QoE(@dmI8-BNaEKEj!I(N*}JFc zZy`QUo%HgCp9yBDWcKnXzg+k@?mfhdC3v}%rHA7EMdd8p&K-U(2%f@uyq20~qJ;NQ0;Xmye0%9f1SPt- z;u*Ta3gleomlX5$H_A;fFqLtIM5Gbf~$I^_bi&La%iWlk&&e=3= z_EfI_$&j;D}X-6XnZ`M8xF*y|9nT1 zt}vH5dYz>asaXGe?LD74jqo0f3-}0E{@83NRws`p_f1p62ry7(YS6VLTe#PGB~3L5 zABpM|7pOg+M@B!M;F2!1%%u{%+B70w0BfgASu80eNGG$x(tlE~{Wl$~b|#*?l%0=~ z?RK+s*WWHUx*-%md5>#zN6qq9BydEy8MwjG8ovt~WAz*L#mA2%G~?{wQzsuXvfGqr z7qN8htc8y{5M_VVf-I=zebhRMti7pNRIs)PUXiMLp~fR%no_CUI7%QZI&wwBfsJ~L z?_cb}d*HY+DqLb4#DrZnQ5h=ou?hz-jAuHq>1)YQ^dAsUcu=09P?ZT*x*A+)Y6Y%y zhSJEx0li@;5Rq;VUkmfvv?G2vNJ|fSL6Y%TL)n}6&O8y|0mXOGH?5TS_jv+?Jg`_b z;>)qNY$K3PGZo`GMw>Q4#U@XUbiH1~dV|||;z{7H5W?&o=SDGo>s><}^K8MJ*jH{yli@4DacX4 zH4fDvGk8KhDk|9L4Z9f9uyI(d5YBhgy-A?pc_GLu^4*ynA98V6kbA6UPXmT8HZECyT_7h`;KSriSv(bk9Rw@>U*Imx+&Xye* zl|1+EwR_xNm$z2CCnHoC+3zS8s$B0;%mkw}mKD5+zL@d&{YmkR4l6A#%UpsPi3E0=cC`)PNQ?h+D0DlKP___O|2KlSp@H9 z0w3*G-pEBt^&X&4vzuF-?clzCQNHR@5zQ8zsu%hb=}^;-}gQ( zX`~*aY3cs+Oe50nd{$YjqE1acWbgh6&y)RIt7qBsq}bB_^FiCwxiQ!W{7k@m^H$em z4oThYEW_sSjR-|omGrdEN(cga;5UEJM5q5mM=qrodohMgnUH$8RPcJ2T9TnQDY(NvaKk*f6HZ}ix z_@n2RC8A~v{OcRdFG8fc{_NXn8}*z-6&UG1-$qUgzioSGwscZr=q|w8#I&8HJl2Sm zpK{uH-+Q|G+*R|mV9SWy6Z)zIu|RpfY5O!TUwPl$N!dE+NdsIJXGuD@#{qBW9h-M| zrCDC57hBJ#KVnn58XM>tfp(Tz^JkM?*KZsmT~{?i9ly&PxwCziG_|+f-)?9~1Y183 zQgb`*9vl=%_;n1r>h}9d-WMKw9xsk^ybX#U)q6f_bG%<}NZyZ8D2EgVjZCu8!6r{wvw0ctAfIym0_AbjDym zJQS+avdyp#FAebHCth#)04qs$8kSEC7)YQS0D+oLnONtXjZ08YLRm~s z8qA%Yy){Qu#|7QI=eL+Wucpyg$gbQ&sPCy8JYUnSV0$(N@kzQFISyAhmT#)LrbN(-nNUU z%?=%>-#Xl`Rv(5cJGuG6&yMuSkRk4@-3Bv|-qB&^kSIG> zab;?8g8mLbL4&rQ^oqD)P&m|i6Vel0F5`EZuANx7RNVexfO|$g6DOzJZU1*xh!kf1 z&8u*?IL^IqGg2t%{=DvEL-X7@z{6Fp;T*@^QDwoKnchJ7DZa;Dj;ZcEJcx|5NzI#j zG~fMYu=z=!wvm_p9RrK~y35Ofw!z(cqcZWoo?2euTR4rO&!B7NGI68D{iT25I9(z=6z_r zFSOlnJIbMz4ww>r?K53CDW@T3j)_rJ*nFEpQ&3RC>CFMY^`=XNGIt(FaM{Foyf*V( z_?%T+<^XN)PZx0wyasc+IsjYk#RwwxW+x6GQ(zzbgNo8zucp=kM4D``!Q^FZb>6{i zU6Zl4BLHv+{|INEm4f2D%`*32=ks7!hg){T4aijK_>yFE^XjogtLkM@`Tb)eKM?SF zyLJC!zAEy3_G!I0G*A3=5}V_BSiWepWa!)9(bDyCidv5&wvK#9;%MXVZ;c8f8kU|l zwnI3sL(T*PgD^jJrTAFqisp$V$1}Ac^5c7I%ru?9sZhvN=o(I@q8rIUI6kV=Su-1D z5j^l&Wa1xuXqhKH$s^T*MHH_}=l7BQB+tcLyjSayVAbqN!FNrCTmz?*H5nP{$Zj2z znTF!;G($s?Kf`56+CHNTkU^*48yh*IqP?E_bNTPGkc~V}o|!TM>yD>CQw@K)>egZ} za+!lJhG9a zF{Sq?%lVh;GPoyaeOPx^wqLp3-4yE3u1b43P=2>p^%c5bpa93d@Z%de=e@qkka*{9 zone;?yhk}lQrd2HUz9#>cpHCiI{RDKIo`w)^-o4!^+pj&ew?gy@ zIK2PW@7M(Hw3Uuk{d3e)jeI}86o}BXT}d#<+{3h4s$yu5sO|YUJ=8YJ$joH22&w70 zF;o`1*IA?*sI2X(KSt0#N@=O9eX%zcN(Xi#@GN1$y$o7dqEe6z$}+zQ2ud!aGmhbV zr0}O&uK{{UJyUo0tAzGl4=?%vPgf2elXp78+P2H3j;oi4SYLM;5B@$+-^;=) zpXpE|uaA>=&qu#!`&`$lXK>1VJ`;H@+*qvGp&|CY9?G=WcRNRv!XyF10*>ea%(@4e zuN*kyu@U1dSO=X2_JQ}sEcfX7VeIx8dhj#6fhdd-ry5OGa^J(!yAbO2SnCuR5|qF6 zW;i)3N@8f+A@q@X_+tI2#ri+cFA{+kSzVcI6tGxvtjWT3H=I9uTaNAfeB>K-FoG)h zFf#3PqWyUwU=O^y_v?o0c{vbyo#@%qW)Sh3 zJNvxY@(Gb~?VoHhbiJ6Q$@MxtqrV{_pz2)xPM>G)^ zhnu@z_{%o1A5D+-qF#ol)j_ek+`q--9wIxH_QL>Pu;vo>2E^s1j>3%W)~_suwNrtQ zoP%Sna`hWP4y;TpvNgUvZNHQBpz3-pkBemGFhW>Td4SC5?$*d(wKG4tJ%C3gdRo=~ z{5YW@p__Uh-EadpaK4Y7$SfUauUpKhpZk|r{jTQ=^HbfQn>1}>)gHYOUjmgF5E<(dSgFVa^J3`z`=Cr@jaFK^-Sawt>gCvZOsi>4!hwv?-`l? z0gmqE7?!Hz!ff3Muk*E*;X0$t(V?Lsuyu7f$(DDiTF>K=u-44<$ARy9ON#;b;|TPl z&+f9%QAp;d^J`0w!CU9`E)m@;XrDJ9`2x zO=%@E&0RRQ|B~U#Zz0JJOK@6d|62HiqozlyBRN*uA3w$>><1O3h8Q52NV4iS5 zf?VX#R@PeAzLB-}>OpYFKe`=LDr}dm$~bJ|Uouf?c@tl>zhAz(C-!}VB=&i{ym)=M z@&-%OUmvgbwy-=dnwf$P8V}QM$teoK-H7{B@9Ps-$dG;_@wn-{Jf>-*Rb^lZ!+VmX z?)m=l=zDE<->~7md;!)UpPCYK+tb7KT)xjfE_S$s+X!B6ZlB$78om}7&v&OZ+y0XI z=PPf|-@fl)eJMB%dsMtX)=$vC3|76Y&9?(XhdLO{B^TO^k5?j@h==b7*PX4rprW_M!u%QceyBG!T>>P)O64i39jsn;iA)yk_5zbIN8e)+_JO&c@G4 zT~_R<88sRsfo7zm^`G~+ls}zMrH5$v)znOYNUHq187sl-F-l-j0J*aCg>@S}d7229 zB!Bk3y+L{{T>8?=)2^PvvoAi(s__qMe9pE=Yah61M^{TEZefY$+7wBNoB<7q$r-oX zAH(Zr*Vb%9T)fQcaVL+8u_sFYAKPtyrP$Ikl)zdZtJ|ON7pM?2Z6+#_@`cB<*#r&g z8ChGK>`svQKv^#b8(*B)PYr62Xo;?3DWXwHRg}eKM&83SH`Zp;SO64Au&FO zO))wyJ)q})(u$?To?v2|pY+B4j2*`(rrBOXEQz;I_43$o&h@Ns(rf1yO1cS{+~b`H zgk|>uu(Kacl=<#QUc3vCO1k-2HiSVG@};VRt(CA(>(=jCd5F_bWz(rpZ_%vdVcbV| zKA3ZYkT8eqs`6?fr%iu;WL`S?3;nLJE8Zz7)2ha{AVJqax$a|s%u9Os?VQpiO*FC3 z6LJQ~?V^mDwB@?4sr~MmwBzt^hVB;u%G>RTNdZbv{xoxp^l`_qrKNnH(xWn`K_Y># zU+q?J&Ui2S03+4xBropBjZ?w*_S*!H4xhtf2k)&W0M=#zw>@1t{IjGIE=#o23grI? z;~##<9vN5JhA2Z2a^KF=ap9NqSR`j@vWv)VFs(*E5hU8Nlfkw_PERH9H{jC>-D zYy6kYT2HA`QJwJ?_uc6Nfyg5l|AD4i5%=z^Kq)a4R7K2P;X-1BpI*O&0>Fp_^^~7U z^RcRA5MFc?%RRTU;8dT*uQMq(WE^tcFTfU16paR16z;EIb-!Nine5w2u@EcKh>H)p z`P!@LULj13k4Hs^=@$=&*>9+D--^~)PEMTIw4JWq9f*Xy90K6Ml4+L9ZZmGC+i6gW zw%yI`Pw>AE$DHMd@tH5hBgq6BO-odMXUvz|On81vgMGJ*j=2P)DWqS?t4ml^X`wUS z7(x!-1P9#r*4*$UNU@454ZzBkYh^0rP=`eH#B|!S4G#NU8~4vefbQCmyjz-W4IPR2 zjeG)fcWK_xVwY<>I}fLLz~qg)mC-M}{ezqSKjT_Fjyg)pL~LyRg#$t=6+z+AwY9aI z7AxOp*yq=GQq{((7=`y)OO;$UhfBPVw{jRQhJp0V>eNEF=k*r@;r@G10lK_b2D-}H z>Dq&|fV0>2!Ph7G7af+Je2aXr&&?UB=XnPTW6SZJtZAmthH(?g{gi;`sXQ?2!V-KG z9AdnS_hZh3OU3&A8RHB9B>@EZoQ%*ka;%cdb|(Uay_t*Hdtx0vj2O3~Ig?%a6P)f* z8q3k6iX#E{{$24!M_;!ER zv5&{aN{M#0$>wbAIxoLfIWQ-iAAs&Ja)4RBt$wokJkx=xRk?U31-Pczwuu*+XG$JHBR{ zY~CC>s&2mB1q|Bf6`+UAdp*_*NpeYjRet48yhH6BFZSC%y>~X=(EbRf4ue^9(^dPU zN9*RB#xu{Oio4fqYwPOM3zm;0+XbIZl<_7Cqn2ih7BwJOX^m3EUSmyBjoZug}B14f*6n`fUll-sf zqlrw#({WW{aGAgh6YB>VM)L?YX9%j`pTsl9gWDf2^WYHAsFH$=6Fe9`&o8h2`L=jg z#d9j!G2HeO5lk}$J*O(DB`4U@k7TJR|E?j_rLwpJPf6+XboM%kztQ7glI6ThE174$+&c2`0_y`mgR4g+ zcoG6H9HyHUAInO&VI7XBW{%bLvqaxou;L?av^g+wzx=-Oxm$X@9Fb-Ut?ok7_Bt8A z>|w{4-uD6895iAsTk&S;PqTJ$qBvf^N=SWv-P3Qj+|STMJeHYsoo?Q{V0kfult{X3 zlT7IUEiDZG;{$ZbKrHfI*$KsBd3|vSfKk1WV*4auO51eoc%|ri&u_)>T?2ajySd?*PzC9GgZgsBCXdD2R`2twO`G{cC(HKl@qEpxeaZDDn8Rbr0WCgwTNu5N20pSEmaoF5iXcA{lU zg{)W`E$z*QH3)FOhMIpEi3?NVU-@Co8b9CCK^Kg^uQ$&Ty4uv?;giM`68`E@U%w1l zaqfPb5+yHF9@%<5(?puif4y+a83ED)`8YWm9mkKZRm<`0`Aa7=>d>JgczPYtV`5_N zWxC|Ot1*~AJhJhev>ODg4b!2X8g|~_tz0p7Y?dD>P!$8ynEIm3#272O_KPr6 z=aWS&$I9P^!GE~qkyg_=1pL#>={ZG`fzfV@Dr+2kjwhfPOwRLZTD5m0R>lJ=B|JDa`9OH?ZrpM$EvKe>VNZn2VE zb>}=dss!{J?IJJ3NPEw{r{lS^T#aj4lly8rZT`*GHlOCnEu=)vnO;?qztjf1($@aal} zm$h@Twjp)x!LNF8F|o>dN177ehxVCIjusZ!ozwuv-Lm_jC8yo#w>Mv{vz+7lwDIw) zqRjG8Yt*)N!Qp(^iDqJUnxe|YcAn53OkIKaogx*O2$@waBsB4kEV+;!&0I2yAju$o zam@KMC#yhaVs}@*XBVpfXM9fA&(7J?8VE(_yYRV?4LrsY$P*9=lVXaV6j^||s)i*_ zQnL_KsDvyu@uj+uJ&{1x8!=>FNL2MZ1?+UqZRrR9eIBy;q1~^K3!EM27xGV|F`mbq zB<}lbYbz^mxVr9_1F|4!kEvTtNw({Lyma+%=)zr8~M`JE6onA;U@q_ z;<1}H2)Xl%sRYnD09j4fY(6}+v>_qP0FYrHGO2qylC>OKg!(mTH9C$1Lk#cJEMPV_ z$Tk7E3weQy8ml*K=f_?>OhzY>ib3Mlh&NgE^CRok2c;%T$EX)3bVHVD_!3tMGSOiq z(WyjlhNrKE;?T7_bN;Yzih2CFn`xDvC6IK|;A6r|*^je-CQHz+b{_jDp%W)^{ry@vzVI z8j-Vl*UQ;t!(;b6tfu2JYMBAbCMON$WKy4ZwEw!D(|H@yd07T|{#W(`?p|G;tsC(a zz8P;FfF3U)vlSUEp={r)1oKOXTmFXy(8DGUs`QUs(UXf)}yg@tD zoT(^qa@uWgY0n9V`5ccTWP^R~IGaixTT?RA{vuCvz%G9ZrtkIMJ@q{J422E>;__@1xp1s*QsUynOlAuBIp8ekm<2Nf`}zx`*! zC59&N^*>wkp4V~tv+OPxMK)S3jt5He`~wV-i-lW*(&GUwYkEVE)8RDxdYL4(J`;z@ zB%hZDTyH1_zAP&nRIc}HJ4TF`f_wg0s)|BJahl!e>0h6T5+Q0n* zmn&EDr;yIQI{NC4&7X^2XSd+a+rJgxb1qI6+YCo{Q|?Ze@Q<CkS=FH{94|SW|x3Dj!tdNte1!`S=Y^K_W^mB zGq*pAyT?d2+U<^&h5PGfYOZ*Dwu(14WK%RcZPJm_@wFTfI)5Pv9Pot02x#T^{li9A z+|*20Z$OmF=;E~=q}5g^ygEAt_i8=q_r1Cfz+^fvgF3w*=Q&qz{Xj-m+fkg2FIS7N z2f%Hppmsl!@_MRs_8c?{aXx8Sgw$E?Mca7pkcQJH3=bCpEW#Vm^9-c@h)=+0qu-_# zs7(>)Ol{}aCyad*zMnU&}Dp`~>*Q2+8)*j*n}UJjT*uBWt?ioMNRAzgu_^z0Aua~3qI-3DP!Z~e78gLXTNo1nnDlI zsV1n*u)f+@tzH&QzRq(j0_&B~JpqNL)#PawUuAsQ6PZkbO+M^4 zfTy$B6B<4W{@qxres{8TbzkZZPLOr1N}3{&~st zC53nya=&8wdIYq5{W0E8oOusDFEM8=SJkARqjkfE#pQFtizp;4FBeZpnLjE|14pGOfj|5UKiN`dkz;>oidl%=}q2g23Mu%)l|_-z6tT-()zjkd%1 zcu9_(HOs8+Sxl$ERSz5Is%M52q#mf6LzDtMXKfLj+3o{5-n)>W0>5qmUNh(Q5xD*r z?m#YEt;X%nuf#5%0o3c=h~B*4zB=DDG-(g<$nv;&UZD6~^*G3%2l73RVaD)3)n84t zn{5rSy&SW>o{k4-Wv#)VEQa<0jMhv_$33SS$m!~l|2%Gby7u|;4)U^Uv*L1g=jv)?$4 zB@7b3WT2WdlKN=O^$2R4HTEj&VBQ zeUw&Be>snfighj{|GV#OfJK0B0kRN3AlW_Um#q;AfWYr4Az1*TowlFz_ zGD)R*m;8lv-}7lw-LZ_mO(~)Tg;mT%t_tpoR?tQ{SpmplD`V(+7_K^KPo zBlC#p$46m0K~dM~M9EAU09P!R0uqh8ksXG|1$ z(v+06)SYM;Qc9aZrYErc8q_ zxvBe&dh6+Ld<_SUkP*oK^!$R*SYd*6GR~(_0A_mWvNGzQ8v&@3rYm~@03yo46Zq{z z8tYz&u;x4Fk-rdCS8F_4a(lY^ec-Yu1tUy9ZZt+-PA*dZbh&kqZyWii@o(P0gZ4y zfk^zE5}*o!G=U)*D1iePz~=BYa|>F-0(wrBX3n8*W8DaJ*WBvf%Y`hWadT&rVYMZ5Y^Vs4 zr=(lx43^2>^JsR_->64sW)c)f+}qd2u7BN`b-^O(ek911)rgJ*IUNGkIS2uT{asp4 zL@gnmu|peNSlzC{cFWkkbGYU%)_U9S;bbu87jVd?sg5x=kM7(LPoF_S3w4e~11E$B zm5bBK5rf+znpZEE!efG@)}$KSyDpBxZ*U}JP@2*5^-0A5@uaEMP zo=LSx%DN8Q>G4}hCTt#tpe$%_PPrzF#2Jq}?dP80!mi)p<~^Ws-QSQ^pp~Uf@+seA zSBS^3ya>Ds`RidR!TnQbYg%>7^bK78*H1OhT(<$`4)mh^c@)s%-_RkMP`n%UKzuP;QCh}&) z$JtSdEU0WFPWhM9;qBY8qy6cAo^I{bTySHUWSP8OV5mmZA!wlJW_in-#5o`wLHE%= zB`}n4KmfUPG9LIHy=Zw!ViE+=Ds~ggllj0xl)l-xLN#%Fr?qRfMPq*Hsq)HA?65lR zY#gNMmsws!%h^HH=xBJhI5KHFRPXi?^*oNv)B~x4nXW=CHEK2j7P{!Q{WNP-GB$hP z{TuJ(j=(bYk0(a1HoP%n^OhH(BGR`H8=i0>F36FWqQmLZ64DkWOxzKd-Q)6-$rYrG zw&YaO-g3Do?}rqn`OJ)};c_3-eQiLPIDgIxDUp>$Dnj%8{Zhs7X?v_`T{$(k5{}edU>!a?Gq)wl-C&cOtYx z#uq_E>m%xWJR}P6JN?{?m_70JKrUO~e?9>mRBs=wRRS!1K~6pn#FT(ty>l$&^6@e% zb?ulBewrUX847ArP^+TL7{ie*WN%mbr&52De=w2|Yxxj)oq(4@^GT8B_f zPBM{X#Y>TJ)W$!rCbAIkAfMFn_f8VPwFJp9ktN{Eu)RTU%1_SOyYVFR1*xY+oCcjp zk~4|N>rKULv>EKD(#G76QxmyuHt=s|_>-U^G(9QI@zZ=oG)3 zLwH)rR6JbN`G!R;rQSIr#~Vz!Ml8H{hS`E4_>>o|P~}C~F2N6OI!Kfk1`UX{Zn?ne ze@C7}alpILh?A8ckxWR;v2>UsLd~Y=wup$o_`K@ez-^w5DdIYszQm!1CzBzK8E;GX zY^3_LrDFI)03m~yV8T_9A6=Z+2b`!k<`2>$?b=01U73I1z-7;*<9lH$8aI34M+UJp z?`xOw;C%qI@jGAp`MOVx?aI&yRKX8IbGv9L znXJQTbaQ4LXFb1g$89TtKK5MbbW)+~2&6bT zeJ(T&sgEy!{9O#J9%glGhi_-3f$!{h{_X3$6ndtnQT(M63w-hgpUjJ;UjjW|sFk3p_%UKfj@ops~@J$n*~hvGus zFLk>PTno13_&{p=U&J7L-NA?#3xIX`_>V4_M$sMJ&&TCu(0*(}>(_3=X4brLl&rZ~ z922`k58E(_Oya}47Kh>C4eQGLSaJWMD9aV2Y?#^B=?>oMc`At9AnE_#kfQ&-Odqx2 zV_ADkTDQ~e>;dKhg1sfOsutD_uENuax|3v>2{|u1|Gn7}`TPH20Tilul-3y3 zKm=MA_5Lohf1H3=$hcJD2xqJ=Ki)xW9@Asg5Cs%JrT49a>vov`dIFBjn(y#{JKJ&^ zZ!WYdAt`w!2rlAM3}Y{A>)+XCE{Y4@a$Gos+MYYux*Z3F4&{bOja-e9y`4|#*tU{_ z#(@vDK$Hbf5XJJ`XYb}R2Xb|rLZh8st@-R(%4dTodAYY53q>0LJcA!#m=X}YsWlDl znSax)wzcUh8aC6ZQ|(X3p6iC89y7~WeBV&At33_7@*R}yl!GPU$(@^v)vIiLCo%0j zAc41@z1kLUP{=+%{N8a@bnaKl#w}mZ^M`CbTTdnJGV8=LQyev1BN9WHUkQ;6H+5{k zynYDJ+{kV;aK#hoNcJN3*9%AiwL=7$qlDjZ1hnCj?O_M7 z@sQ1PLJNPy=pg-k8%QzO)tPN6f_JbO0cVM=@b4ry1I=2~9sc>}U_C0j1TlVi43m8W zQuxb$zM+FznNdp|Zt)^H_UG1f#>qR2#RLU766!uksQC^f@cAA@iWGq{=hdmZEziA=8Fs8gx=sAFG!r2pckrCXZNFjtB9^{UD+6oS-@`00d=tR_>$aMHui`UAdpQd)+;IA%)mod0+ru zUiCs6-GfikDPY}tYx`kVbu1d`uyftdxng+vZ3cNUbq28_{v8zIan^Ii>CBI$Hl4}hSZM>U#5P`0E4F{b za?Y&KzxvvXlnO*70EnZfNSx67=IXWHN&1R=eD%!Yp3T2DEJa**y;}OTOgN7*aZ(ZZ z?i5awVJZ;qb^Aa^g;ZL;)i?a~KZ4Fi<=Nm69;GrQ>A$nKzD0o{8f# z3R{c{wT#Dd0bg52#ps=iZr;rw$)yqkD&9KO4@78wPcqv&6xV3LlU0|VoMFja;{y{? zu)cT5Fu_ywBm}c(H>)1z`idz-6DjJq>lDKBRv>WiY31i2ni#aKi1go0)m?%!l|B+q zUo9opUR+ox8n>-PdK)GB%%jvoj_%wY+NYP6mh3T%8x>W1eR_;=S|-vt1~VNm2YzaS zs_H|9#0K*Ff`RyXueDF3De@ONJP`M*?nj?rri)rNv4ZO-?7KS;UgaAq`bq^ZTvYph zDNOvMH;X3tqIX;eqw@l7eMTM3!FR|=)jb~Esk}xVB}BMTBC}Bc{_g=GANbV0q|qX^ zccEE(-5hTzKNDnsQcwycIc9c`A5Dz3-^>8AFk;v)Ma5HIdWc-l{jm=Mj%p9YC4^NW0(gj*`@V687(Bl${O#>5s zs|6_sP?34bkLQ~C9;?ETcJp4ZLNE1AR|98fH7`GgfvjI8F@owDkyXL{Iw0v7i zwDa0h2Zf4bIgwG(Da%Dz29hjg_AeZkUv*fWRtCZ-&Kpxqbv@0_w(gFJ@bL}jU3zMs z+IH|%wc3?Ayc)*e^t?gJS{X-4iJK-EB=(sB6qi2NejeyVe_R03?vZtj!(eDq1 z#(ZVieX^Rsvo@8Q=n*Dt;};ivp_cO^R{yh4_Bp#$gdj98Kiu)8h{Wo>WNpct1u}8Jo1Fbm<;P zO%h>(vi#C3wAjbcO&-9pjF>}=m>5hHh8sy>%DtmSW_czN7kAG)i5tUNcYi_qWb z@WV}?DzjNr+f7EV1$%fkqW!WBw-6l#xwXd}3|c1qdwZU&>&g3YapycB!e^JUCNDR3 zBvNst14)X5p~4vBX&B(_$jghC7<;-f@86hoyoBZ4*k3hIGZbY?6KPD8qPg{<<2hU# zi4Mu-z8Q=J%Z}4^Y>>uF51*@t+r?2F5wO78*_K)7hv%8o2i}xLBT9O;D%zn_9iDg1 zCMPWoN6}t}N0yJbr@Ju83|e+Q4++3(y7PwxhEOkR-A4EE7NpUb*H30SMh zwB4lb&zvFzcYbAKYd0F68U8ftUe5+78yozqQZ?OHkdHvimm_%rh-8$MzKGuLE}fi% z(Ck$kY@3m0T8tPK~De=&&rBHj@ZBHw=M9E ziY@v_(w-=0LrO(cq4|f(_Zu-59mR@dCRhXSZ*>~bQcLCDr^LnCrBbwvA({tZCvG!% z7GOxEF{KU*4J)&cM3{F~@LMY^uWfAhK!S zr8qddnAFr(SO&ja>kpa7%J`To&(rZHZ@1gI*Y9LujvQddMsoU=u&@<2DTmUf%_u@p zj)fVuaU9LOLz^TPe21e@h zItl?&(0m_D-w?uVlqims+M*}9cs=xC%4=c&cXp)YSGUlA`N;6JBvx{ zWG4^492;H}2~P^@OC0MEp=ggH!Ep*2iB#FzC%K0}>Gn|RYTEOwg$`Ql=$kF5VdbTR zYkuHyOnh7%3sQN0Ex}M}jw+1e+*FLlH^gqf-KauJz5H*_E$)|mf8hpb0;aI1i|{z! zTF44^esh;&ux|&j5M{D)&i;+)oOj`u@!Ulrc^f+ZsvWfDs}a>H%_}83mat`SnZsR@ zH61u$W$jAQg?^8Xgp26l`Wv}GP6E$E;dE(R!1{L}-`n?IIB>|l_~{yl$OT-NnPO$8 zUOlT|o*|8*j;f|jQ{Vl`%W`?#(U;SrjQd!VAHU_*LYZ>eF>P=^bo3Y~qzMSf57j`Q z{MLBY*sMsJ3JGH~4j$spLkMmb0rLZL2W>PFagH`xum97^;{K4?O#`_0qBhLntFirN zzDs56d9X^}Z1F-Q@t=7g!>ew4sUk@4h)59#?KBJ+y9RGou9&zt`Vw?~eqPPW^C>=Q z18^cBMNy7TkX!np-TVXRiVN@x$_Z?(B+;vZ+~#?*eQt-xnL2Ac`$u+0)iPbWKixq* z?pCy$jYsiKv+n3~9#+G`!uGNO4f7!B;rK+gX8Z21*-6IYKwb9@eD5kix^NfO)MjH~ ze|b}-mF;~Q2kZ_2C?i0&RS#4#K-GEihf-jxJG_+t@6LWsST7?IC&k7`go&bD5A9o9YAz-6U1R~J=+ZN}_N}x~)r;Y` zh?}NDhU$ONwNYeD4Enw;Z?loPiUUi5wtTuqo?bVpq$w4g7QDH-9swim7b_ZfZ#`zg z^*3U0stbZx9VjBQ=`0r#7QcZeloGB5!#2HY-Ih#Nmp}woV*ES7b>zLpRCQ3JXj4u^ zdc|VKmZdR%EJ90CritEwKrPvOzr5dOPKH0PH?Dc{*jh}zm|KimFcqsv#Y`aV^Gg(j zo37Sm810mP_P-KvPqo~x@sQCExsTbZL=5a_y!0;8!@oZ!eQ~CL${aY>wX7w}wi?kg z68h=fS8x8Nkea?vi^{H4V+MP29}QquRG*uW*E@hXJd$5J?*YWA>F-Ojp)-J)3Hm1hV4WX%gO99T z7kMDhc~=v_9`q)ks{3Z(iDcbzv-i$^(52(D_sl0F#0?-T+wb5=^A@nLXh@;=HdwO~ zpqCy~(uXC!r><7W-3DM%g7%U=HcUR1yfPUX1=iZ%{AHW?_S$Sp!sDt(`i1P+V$}%| z=`s4=bq}K! zNuHriF?EIItwn-S&f8*$B|JXzr}rT zn2S{W`Juu!HI(`tR*@f)^W>_Ftt@!aWA|Y2-T3W^&G`~msk8c0<4J=*k;`$Iw!Z^m0)+HbZC(7Sqis<~5pp?`~%ClY$*fUNC3eX$ z8dk^*NS(*%l1r3?=86VTRy^H`ri6o|4E%Ofr{R6adiZCadJs^lD-T7%V#A`hFz1cK@VF*ixBB7S)2G9?o8OGLn zu&Y)b3|P;bVH;AWCv+hAf|B`)Erk1j?!+%JnWN7-*CiT{a~Rx;()qaF*ujVhFF$m1 zC8&ikyc*%Y4WkZcvME(|fvJ#0|J9-^6`gw=J2Y=qM>MOkdB!F~x1Xc*3qimYF+)ozrzRl( z4LT>B?K%_e(s|a){s#yjBVPnV&VefO!Kol~*k(H}a=P}S0_>wERhhIAY3%)P-?rpk3i>!(7NLUXIrKr!Ac#08HL&8;capR}-q%h( zF{rfg-*HhUMm_rRDL)j2j7aF6cMz<+S;wT1q4~@4r1*|~;jq$K0??c;qDGb%>>28) zd)F{-$TEbI4)*<m_7(#YyK|J8@j#f29aA%Q?Nu zpN3_CY!{t6sE;ZZu9CtNKfHS2=jeBA{U?;k^tZsflYr~ANr(P(DLao)@@QV3n=hf3 z?F);>0@Kn0*vdp<+07SZ!{XP2=0*Z1gfqwDYylzk7`ugiVKvZaNf78I(8i%FELaj_d zVCgF~5a(~uaZ+w+lLk0pLR>t71YH6gqVjgMFL7*}migDA#il?Ovgn!px$V;BmWn62 zhl2yK#LA()WAFP9rgyW_{;n)DdvAYfED7b>!|$RS?x)p^mrxT01slbPgSkHFa$G7L zq-kOo?J9K1qJ9(O$pi@9=L>p=Nc!h+FVcxZm(kRQR#xWv>_>hU)=A zPi&q)1noEigpL2<6byhx9SCV)#sjEpPtUr&s%Ca0mP!-VH@Oqit=z$&hYJJMCy83p z=g!*C>OdpV8_w)B$cGw#iH14XVLdI!UVEnu}5OK%F7n>)KjX=Yqq1k}nN-3c|)KFK0>l_6r* zTAd?Cv1QMfpdPc0>A^se>A}oCUK$)LGVvJP6uy7PJad=l{*&v5_TsuGa4luWEQlPc zHxLuNhNeC!yuClLb)M`xK4 z+!b1V-uK2LBh%JB+YkH#YH8xS^V)o89iQz|njb!Yu zZhIIgOS1I2f3_jw{hg~-LW63@nBtD-+FV&UHqV#rZQ|?>jsG-tlz`jqR(SLAvj|zv zYizz0jtCGUDGy6bBPEy>2XS9SSHoF5lXr;F1M-ik}s|-yN5OLEo(SK{ynAI0Z zBDj%u^nTLNU^L)Xozs3qfB%!+9tc8arcHd+smN4}4J%8uk2kVDZO%FQ>$y5Il{q3E zLqY?49f4d%;p&8tJWI!X%$>9q4FZi7U%NYiUY;OCypF!~F#-y%6?!+oa-0;Dj+ery zJ&@~?p?yRI+hoW3FSf~Ig?81q^t(DZ&qF$tS9nIK?{RgU=fkLfdY+~YE>!ENJ=yJK zukyH@xudj*J`nGDY}W;2l*%I~nVlc%Q*?YzoORZhVB%FC`7i1`5}wt@pDI-orq3(0 zAwzr~A3sjNfvWyX*9_HXfM=jF{jnbimp}-g{8{`-K`=SC@gQ21yI`GXM8a8;?cdfZ z5&MOig5HlA*Cv6#Oq2>CVf&-zrm575?<1Qol-HUAt3L6?c}3}U8cn=J z&Qt;rvb|acj|Um7F@esWFuT!B{8I@|X8NhWdkXca!#~MaX#n zm~uasrS56E?H$hhnYiFAkBF^Gm{IJ(0qNLvWPP|40orU@RP>*kKge1iS%sR)Yokve zTg8oQNDYf0g%7{kYSsPX1j1A!)ucr`wa|$nNrZnRJ$pdiIdw%ye@(BTXKn55Y?K%^ zwswPV<9Y1E_*s#Dqm|o5_v2afb+{$}bwm!=V`c>K7PK(oYo^CbzfF6C*J{g_5J8ji z)hJ-hr8hCNZrgc=bZ#9ztwd!^Jl#`!x01bHKwjkmkSw?tJz$$RAd*J4N>^)$ZKB1O zF30=wql2)qI(jE*chZY!@>hk^D<=HRke#uyu|I#*W6mxr4jE!~37$CP5vM9$Mtjo1 z10U%F%Awd0sfv&CF2%Z+md%-*T?~9pvg5glth-_Pg;}x^25hdRk>V1&w!Q^kv^hu8 z0$%-UGgj~7!L^c1`GS17cd~Ju9^Rm|f3+U>yH2mm79kPf3@$Ll^pt($M+a8dfgj@? zOas9v!a1Y1FC!tyNJUZS_dIj~CBha#Fk2Q#>sDV`XV@zh2tUg3ceKxP`8bkpX_B6X z9ocU@Sz$tKt>6$96E%M0UcC5Q`(%!jkD-$PfLf zpiGYK4?c(<<4sb|J9Uo=Dv1jQ4i0&d7WXejC_}S52loWpjf9!ZV}vA@n_;@6n;RXb znx^(htn=6UWG}jmt_$qLUtYK#p!!LDUvNW+2$?LZ)84M&7g?ew3Hf@;6=i}54zQJ| ztgMY3jRJoz*WGnKz0B@`>fL9H=wp_-6yR5GFUKO#Pu!leb#{zK#vEC0W->BPa7(T| zhEkL%QL#O4PgkBsP=G<;L`~p^hl%g?SlhU5WwacQ=@0NTsbi57ucEFaeHgZ$f@3eEx5P6>8nHUv$!N?XHQFOFr)?IRJg zXwm&R8 zY4wqv5?HkvM1v$juV!EKtnAnsu*6bRzWcWxeKi#sSFtHyG9vv~Ejn>}cDiPegTtG{ z2wjUxQlat6^FI%N<-a$bXw-~P$`j~LcDCS9D&E1Swf%b9O{#A#`D0fI-qm!?Q3(F1 z?J-FY#^MfvH(G%28Z4r<$PS3JuILup%_@o;hN%y3Z1$}AziVfwyTddKU_WXt-^t37 zA~2`9Byq^H(Qwx@_vlYiHA-8nwW;<{tVaySM*|B-@3Y}NlEBENHx1;`=0uxBn&?oHQAX~5pxorex@(#Nf5aqr*HwI0qRZ#VxOjo)~*k8m4Trz+EaRvO7t zrd!0>*zqq@3s${92EFX%yq;4}a9y>rO>%cG9z6pdr04q(m!~72M}ZmUprD+`W7A|t z4d;inJ%DCuf4Yp|ap`aa)bLd^72sC4s|cSv&Yb3lzZqYxcu#sFz%=YU&Q`5Q(LUb$ zc^L?`h#yS(l^Or&thk-AGQA49 z>MiqKG9s`g1t8E_u&@jj9oS~SJ7Bd4syN8TL`U6t`6iWZ;je^>x>aTa`eqeKKjfH;O+waOj=JBeSI(9X-zmfvON!C1-TPt-oKS#KV)Meg9pHdkCk#QeJMx?N4?&V*8BC+*>-x-vukRJ~7PepG2{^6{Xs zRf=qFB@9;bH2$6icXk9F`W)T8JSNEfbsco+bYHydFwRmeJWVx>hBWz{!AwV z;NyP+R*^cN{B6d2rxK0Cq+6Q=WG?Zt_x^1onc&VV_aVX96(8T2l6>#=!2BUB0|w~; z`MXs%j6u98qm@#oZs*T%pqB&kpa30jtIG*ZC(_c#@PV_D`9Y#@T)k%VJ0*;N=bW!l zWWM>nm%An&_?7TRu~^ZOcYHjYXwXl@6zDQUga2?(DX}GTEKU#~&?{C+f2+E=gd3t? zByu>P@;g)-nwCW6p#%B%sJ+vis=8(xsNW^To8C{uCO}NEq&W6Ykn!DB6_@zd`G;D3 zs#4BQs?}wW1R`py@G>kZ+)?b zgEyR#>-Fv+bpL+a9*@yAy$@YMM|tA2LCnKtj{R-nwG{1Z#z9g>$=J<g&RBo7aN0$aNkF}ux=G^lkv6Q^k3-fr%@blw!TcZ@5b z5&hxZ8yiO@%rcQ2PzV_w9tI4}1Xx~q^I|=eVA@4XPv8B6zx1*CBX20JJU!bu7jbi* zEkOSDFcUv%=y{yfB&1K2DE=Q?UmX<3+U!lx;1=8=KyV1|5HtjLcZcBa!QBaqySqbh zTiiXk2X~izlXLF5@40V%UA4^C{z1+3^wZw`>&d?=Rfx&2mScWDM;BCZ3L<}zEk$Jy z_LkovdawO;wfMAQi(CwJdruVtG#L#R_s3IyF*P#{7yN}HR>A@7RmDtNh^QI@x6MZU z*uHur%^w^kcNb)?OhX@QB()(~Hwk!cc!Uo0VULAw+{wVadqzfPW=6)FDG4+klr=qh zA}aj$@6tz3R&77LG#q85Z=*7lORLm|ACh=VJX{{bQ(1q0zC6FJy%m84m%}GXa^pUg zFz2C`9jgJGBmsEG&6xz!?qhqkHq_|M6%E!CpwL0?YN8i;B<9f_W#+bWJU1OOJ@09f zj2bRiNL}JtZQMbsG2o{z(Q6X;l#kIib_u8KYfW!F-|ENRpGdQUw& zh&|_aLndmz8v&hc90I!MzzP{)oT-$wzipza<09zE=h9pK8rO7L4V2?*JmL|2)yiOzYUME&C0JWewZSwhVrYdz!wkMqKepMpkyybuQ68g3<5uB1 z3nun4ipd8u1?qEiz@_y&YF6Lz{Jr_OM@3*;2R%gi(jx#V%-`yYoMUYO)Iu3Mt(TSP z+`PWBVLxC+5C#Y3dv)a}Xo?2qK&>ee=#q{d_5q(7ZATzI|K5LSzSdg6N)T%*oYC}X zKwRQBF~SRuUQ3crmk~m@#S%;JW;lk`ajfbPW3h5C6IZOuuY*PIwC3kgZ$5yB7=Ru; zboLV}CFHFu<8Dw;$6=j{2g}svnot^VL$(}|P_=)ZB7BbB&;T|*&5ILur;32j`|MuV z2qUzPdo+SF4MM&G^Dn5}S&zYOzZ|2YqktZB5{2tX{HN6X`)sG3w4Mh(x8lSF#Uwia z>WXCu+fNI5l-XCmW?@vFg;tQ@GwrObNM8tikG3{V z5vZP!68c1k;?r(pqE44a-)HDHBB{7`{<6cCqn zUup<%r78BlWKhE*^J>XTAhZ1Z79mx0*19!^LMP;d?G^CJM-NnLn&tJ z+2;~wZ>7=M9JSjh0Zn*L4lPvT_7R$D=OfC?LBn1NV_XL^g*5hn^PqS_nUASr?q6@6 z(!TZ%zFsdwMhLA3#zaO-N6H;nnJP*Dj{k#}wV%rteMp3Y0vF@s!>4>dS7sR!B}Dlp z3+t81eR{<9n+jC!bAtY16Bn2E(MQ8U?wx*Sl3^IwJNeBO2HKJ}F(H!D-^H2Qa8Ox$ z1s-`d8mB&N4GweHquNV7uCluVeqJH&`p71H-B<8Cxiv(5oo}f>9>eH8MKPg_MUm0R zph}TO|A?!a-68I_H3mA(yz)RY8nwnlYpbn~FT_a5K!O1nf|wW|g%NNh`RZF!NbLk` zoh`#p_%Od-TKQ&4@Z@ie+TUaIMD$j1{`mR+`AV;`%RBn`s$<2~~6_W=frVkWD~f`q5rlcOPTPVCuq*vE$pc@`!e zowi5OsB<9&-%kp%1);WHpM8g6#C$H6$@f<_5l?dXn-j@8W8rsYf6V3w}$OudtO|ULXREq?;t@6=ywM7K1$QAi#U7AyY52*o~ z1Iih~e6rdk?`{s*5ZyU|T#V6_fwi@@p_kWsrjGY|vx}Tp~4S9Iz3Re#>h3z?cgz$#RpcYr{a;a>CYnG90k+ z%SFt(veHl|?u^v`Wocd{z?Zw^ViByz=d;4LJrQpMttE zhu9ZWhr|okSqY6)TdDP`Gf%21Ctdh8HA4Oj{kj-fQ78Z{0s89j^DAmbTD$3^-_OU; zzMLVKp(EmQtfr42`fvhv%zHG*bx;)Y))-raDwoT?5&bnT6CkkF9A>z5- zq6A{{x0D~aLe>%*Nhezf^P}kHY<$?>>>QfrOGm9r#KvV_#dGF5 z{^%CRpRJR7Deq2Kza7-#8`fq^%((2puFb*6p6t@hN6sz27{03{Hd|N6&(Tw#8O*{h zqA?%bGs-H;L8?;CK}ePuq)_#am@C~tUl4xo*>#bG5;XpZB-;)nao!37T-J3^dOH1Z zk+bHFh8ldct_^DaXS)xndvh?bG5Hx;ZfMJ5Nr1|U z3$?WKc5wL2lGtIpuFH}=Zg4%$b8M}~@Lao>O1pD>OrNJj@;($vy@UpC?0}R2DeK8o z$T{` zBb$2bgo;(X+?gNtuEMM;(3rTqkWPS=SBEW#I$8Q%Tg)7EEhZKC;I7+`up;05#)BJ| zPerPU{}n_t1RIar@;y(@k!;cF471!t+Wwkm@UMbj1WJL?a9k;~&n}nBfI&l}+Lz@Fi-buETa_}tDJN?N+(*dsHP9LyA6AqR5t=oe{ zGjL?!6(399v5bz7ze}Y|_)=;cb1lr6-r(VGZXZkd?1O85ikQn|+>pU7c}(Z-b)Dvx zJ9~ZLG$C6wl|;i$O8Gtz{eYJHhx=d`&_mH_8_qCQ#`zetwvv5Ep3x+65)d)uwOBi1 zj0(AT{)QE0j|FpoA3VGIem)%a^A_X*Tx{-MY6>@-cWiPXL?g*(PsWjAwoJZTlgNOi z8+`)Qq(a=qyj@A*@!-f_xD}BCFU##$%FIH-(S)71S6iJ0Bgr^0&bF1DybD(BTqXfD z2BrQ&Pn@6l6)kUs!1~O+%-`b<)AVU&aO)%La~qPqly;U90fW;Ybb;Vl#|n3QCZb*K zUr=27ok5R?WI`-%G%Z z#5xoY@9S;t>O0WxI)4Q~E$=@5u;mL@#VD!Tg%sakd#N23H-`%R$x0yV#Wh<8_^lBb zVld1=IKsi9I_vs{iu&p0wfp6%5S1t}NL&LL0*#7}VMXM`Bh4^HP`{$m zUL(4K79o;II)eOy3k5^#kBmXEv#SxTzYDqQK=pvNL1K&!N0}QPb*fq*ARyBEVGM7S zXDxcVGbK^@s69(isOq1XD}j0R4D%jwq`6Zpl_Z!Y2IC}-B}C#`IXg^Es=`+4W)&vR z(os!WfAd9|Ih(TwYVi5O{-u7D+Lc%-k4cVl*sOVqg8khRz5A*^@+d{hTg1ei-cA;4 z2yX(}x|N~@iK9=9u^k{2Ch@IdoeqG_L&hXeVh!y4tYfLCdY^iQ@Tqx);BKJKAI{}J zi@F$xk?qTXpL4I}mmXBhHEZ5WXbbXX`dwUvU4(@J^QWakB%684RELMMf_GG0KyXZ9 zbdkG$l$;fKY{YUPz{)=+s9F1#ML>X?1=t`93m9VxV0;f~I8!Ds*~7@daGcCADt;|Mvt$LSCC5i_46|;yDcbZ#t2I}9ktE= z$py@ME<3#^V1W2z>IQ!i*NKg>^CJI=a`5nIHY!}VV*ER7pyn`74@}O=FKwmJ5LqGQm*2w^F=H_#Nml_!;&pQr%Gi;lJ424 zrM;fqx&YoT&jl=?QtR0nXgC~PF$S#BO<1cY{4Y2Yi#HAD`)#_7{W!b(sUE) zt4rWoqPVShdm^Q!C;uG%JCgTeP$P-Fa|cTGp?u01ar>G)BIARR%z+^YA8)e>^w3hD-AZ@+5@HRn38 zM7RrP_h_BUDY7vTKedoxhQcvWy$l z2G%V6LtKVJs@=Ts_0)zps$E+@INzU;5y@P;usxLS zOMUDXw)+_HiSmr|jfba7A)BXhurV=8IA&S>yLQ=yMq1$>TGGaE4Bvjeu|~Ost^KgS zUW2ZS*h5?Y*q>y~xPa~NW$w@K9Z%ylMzT4&xAhs*MJk$ungdE5A7Z){&d}X$rOPMZ zKV7|yipSNr2Yjjg$NLGjxPc74k*Y>Yb+dv8opkU6}5`rKR zii#siT<*e0&Kueu9D;51z$M5HcBX?i=9DO}ba4Om2eIQH%7e_E>2+H8K z>4bBl8`cI|&lIwK8_!cnrDpU?cbuG`#?)JhZY|9*pKX1s9`4X#M6Q45BgOqj-I&Jx zcd8XjOF2@BBgH+jN8~S&YaPlTaT+9W$bK=hP1>Ew|gZ#UIX2t%16pfwc9xu zuD*w93~@Z#a1Tj(I4wTroC`ZFZbs+3YG2*TO~tjfP>w{?Um?Bs#gl+AS0PW05S zwwVdHoPy5zgOC0mVtyDxO}RwbP#ASaTKbdnCU1tx5no3G`=Zu3DW zMUdE>bu+P%*9Z+vZ!cuv?MLK-&;_xmyyS}~!Z$lV*;)iXSz~H1LSZj>9||$0En0kW zYQY89H{-9plPm8I%R(r-(h&aWX5d*Vl!!UhHcf>34Oha9$}j(|WVn+{VDtD#5zF;= zY_*XZgzrL3(`ljxZuuDhuI>8n{3+ISn!PQ;rrDpIn~9D z*RB+Oz5a45hpsh0FA~%p2gR4G=;)+>u~I|fl&|SljD#p@lQW_NpUf3v*w*UvqYTb4NufuIF1auizTeh$_lfz%V^W^|VKDq1I$nLiMS8 z-SMJWJZ%(BT;KMlN?WZYl6@f;#_0^T_aRn5C8LRl|W z0EE7)sR^O)PW9sElH6O|CSa_<7J;0?hj#Q_DeS4-ue%@%p^s~jV9=85gUv%ZXu7E> zoF9_(1UkTK8L9F(ks^%)E+k;Hj-0(XvwPBp4i+`iNJzAbo;v-oo0CebZUSKrYQ|&y zCMus0Z>W|Bh>`xasf4@rJC5TI-MuQE26h+>0?1arA0xp~{UtCU4g)kuVy7@J?Qks*zzFN!f0_@_=f%5+F3QLoEMtHlvlt}&l*2%du1VQ|FG;a0rRJ{ET z#j(V6W&jXq<&b%V7!9NYDDBtr0S%Xu4npOdqdU0_r~BF56POyhdw+Sf|0p(_ZyqU} z=7LzJu2Hn(ghShC-jw}TCeC7KYqXh}=)reNIfRl`VXSn<*K(_hEF>Lmu{M9dK?0CD-V&r9SH%CA2=}$cl5fm0phn|;!?R`)3hWLw`^bn-R5afZ$ zr|tdGIo4b=DW<)XV(9+3Uo6GR>0&pdZv%AO;{j~J$y=+>o48YXY7^}%+EGMl(;^Zc z*z-k&<{gfUzGg%kzjtuu^zVpiWx6*$KoaW=jd4W^^C?%$QoVs<;+-3C{}Psqjxkf# zXe$IcvxscxVPBv17HDm%+nd6f;w&FW!|`|T^|b;W>u;bEU3&SZI9v4%XLClJzG2gR z{obv^=S@`Tx`ZlTI=_4mQA~|z6-mgUCVLe{cRI_N zey4ut`uRPT7)zTY2q#%%le5V0_*0Qzi3v)cca&J!S z2;%Ob=ZZKg8)2<(9xV6B{P7zBntRqho|=LC%3+ca%S?0vsePn|lGAlTU*>x7lgD{* z*pWzT_lwx!HN66fTuF{CC}Ez+!iSSg_)Neekf|+13Xz$s^qYhytnnz^-xZe}V)plE zt8EVNk%M{03ODEO5)(O594YmIi}wkO`ht9HWN-q9M=49#OSKmy$Wr30YV@B$s!ys-#er3%7;b1rr zD^>5e6%m+nY*EhmrU^wib$|LBOZO;r2zcs0#jJmbaEY{?tSUXEtaZBaJsLB^EfrsN zRye~@H3!w8&EqTp@@iN~UOf>$PMnw*l5JDctF`l0M|NSDOv2WzTb^l#Y9!c3w$2Op zus=9KFyhTv1PIqPB=5%uu01W3Oo9m9kb)2?tL&@fH!3GfkEkE9DPdcRuJ<&p%rrk= z+kGER>BoHjLP#8oG>AYk9SB{U4dcN5Cw%)KvUd`C%nt3qsN+`Vr-|jK>fo5!-LQjC zM*JLn6*htJ!YFvq?M_+Ky_fw|gklC;^}cBU08npjLb$sGI-7fYd7-occgri)<8g1tAK(n5wflSR29et-7-oCe!Jkod_7ubJJA0% z=CECRk2SXCv(|dxh-V+UM;6sP2ZQT?9lx~R8&@(1IF@4Dkr+nQBrc$99{@<+pS1omvj{w98U}MNdzQFt$dQ3Fdki3Ln;v^P#q`%pmhC>EACB0`BG{WO9LU5+>b-#AEU5}a5JBbRM{ z^2XxfTPmp9TdM0SNnPt~pI%sxNdNOwYO`m$U-=VF$Ulbk&lO3~8gQ!^pr@3s92vxZ z#|0EfE)|tExBcioStAkkNSuaUO}KHW=30{LU)B0!NC^rA=r(TjSUi`cAQ$m}zfu~D z5A1xm4IgSj4Ij&SS)&aU=@FnC-7ku7n?RU7t|M0@GYz00AINBEKmw}krjD$+lLBP=YGbz@Np8VqUk;8JH`|geEC2scqMJBkSRjWb{)cW4N2r|gj4*IHiK1d zqRnG6@AJlEhx})LVx*EAE?;SwS*qKF?w&Bv_!u z46-={VGsjZCMB2i=-96#uvN53LGyGoKTj-`FRsTkqJ?sBC z@l+egWz}cEa^7z=M7~|YmvWdzzFjgxOpEx3*AeJRA;Zyyp5NU%Opo?|XsUia`nRY> ziXpdh7Kz*LF2e3+=z=22;=A>m7HCYB)D;KV2tA2x>zWXD8*QZ!s3{~B#&;z7kpK;N zEdg(j-F6~OT+K?hV5Jn(J=K81gt5=SG=JJNHua2t_Q7{6CT_@zkFDSzsnCSoIE8V-v^fE~x8neky)~>Ppxp9%zW;KtYwlPY*3>ut z<^Nb7c&DB7R36yu$WjP4n}a#z_Ei^2f6!%(d@Rqm6Sxw-d?P6k$jz~_u`8TX%k+g7 zjwz(SKcxco(YPAd?1Guu&qu)yd*?t7#yj^}PV-itd!>sMc;qgA-xZ@(}fffO_?R zh%hNu83d?<24&w!zc%l)1f5Q$m+)-y_jR*^xiwn2UrRA&k`c_il*b2kZaPs}x`_XaSkTS+WhdbSv_*G;VyoRh{Y`FMI%wE64hf zCL3&5y9fK7{0PNUIor}Zic2nNz!VHNw-CqK5R$F!aJYIbsp80X3LW>1F)z|h*>|ve zLWfwIH{BHw=`EHqS(k1<4rUa1B!pxVmL>y=jIV%UF z=x&^ag$ro&)Bq#F*~jvGN@y1?d+z*`I>SFZK5q2gcKr56_jnL8a+Y zO$BVegz`Ol2(^B{Vlk(qQD(ZLjxvG*mf<{m^dfc_g+P1XU^mSNE!mXo4>EfKt}F9D6bIkKIq z#k)wUW$c8LtkZ?q#KPL|Wu8`^xC34{e~Qa66R9JNmu?BLko)u;7XJ~r0VfEg=Rb3f z{~|&xp&7aU3k?Il$T49@J8p9nGDg@gFQB`)4G?PcU;T4<5aQmdrQJV%3DCv0t7O|` zVY&BzyHyo7{s!cwnkNwS0TlJ#-hjsvh|N!^;Ezs+b7XCbhAT%!TbB98(OKbc$DsC! za4$k{&&prQ5a1SkXXN9$9`ZlhBAP8FBI4mr&wd7R2R-JXySKe6u&ii%N+$L7`s@AQ?C=P5Ol-PdcwTPpQDkr#CP-YxUP1SW`I`C~LR zvpOy*Va*^h?Y}G|9$u8eQ&2Iw%91_jeeC0v@oJ3~YA16|)P$G3b0k3x^<`kYiqOxE zWi!gecS#Mz#7Q4DuyIu3+>C9Lr~tIKhqv`1jy*!yx*AlW zk1-x^kFhjA$S0TETypnq$6V^SjYR&UZQQ()M{!E&{dn5V4O_wH~)8`x&? zZ<-Wp_>l!49j_*VoQ36+=qOLf28Wlj)d1{t_nVM|=~|Nudbm}xXr2h?jve{rS$2P? zF$H*_dC5>bo?Y3twpb=ALXUSe1_z#Hi8Ms|Sy<2^BXHD%-afQFufrUhX^Ix&pz$ma{=^U60rszxAv1+`mM)9Sl(xeM5!Rpy!*XT zW`YU}S=hf8xcroSzTFLg=sq|kb1I1}@fADWX(%{;DyRf zwX{p02-V+-JH=ctSPQ?GP)pC%xScr$F?omYp4^ki$xhdW{I%@o|0szCd4UhZN}|$; z_18)lj&B`y5p!X|UQgxbp6^}X*9?pVAYQQFofhSC-`~zzKi z)r%vnM-Rlb*!!G89ID#^E*H18E5C4sADfsv?wn2{4(*Uq%*YhG?$F`4Zl})o3EESe zur8ZcOpP^)99t*4=Hq@mfK2$ad^-{ewf4WB$xPq4Yw@h;BdP$M@wb}PDVDE!Pstz( z8~u_DT)3TzJJq;XIsnXlaZf_W-q_BQEA9{;M-?-(xk)PF&wJh+2+aQ(bcNRPoxr2# z=#tb4hH084sIjT9H%+HGPWZ&lHQO}!dewne#6#7!^GAE<>RS54X26d077-qCQSmjP zwf=j=>$5sYpwtH>pcY^0=GSP^ReIZ3=QSqE3_uR8`7Eqc z);DO2W-#6Ybjpw(FUr>N#EccSUP{&|vDFg@E)5e19~e)dx#vudy3cne1qi!WRRJ9< z&q&|Yq6T=$x!iSJi|NMpJ+E~1;QrRw=HCcluAz`O&2DGUPqlqlV*}lJt`QIkbv)7V z9?3K4zUO0aM6c&w@cjm&T^c$E?c6fThlYY(nd1gB$)Df>PYF=4#Q*(7IlZ@t3fqR# z(9F$*N=HB6^81U!ozJS7OE2T%b-DyBuHbkQxdQH{J@*JX6XC|2^~86wwJW@BG&kfh zZ%!{X;`URuO$kZ-qU^JjlrUTsMPq%c)Mb3~wJZ8ftj0cbl9_6Tz#?a=ja@k|;_`X^ z*aG^@ANCQUE@n;sKfDd{dn``C#fRgUMlrngw|o0k9(oU&7D}D&}1aj~F4^a8j1px24)0f)j z&H6Vt4sU|I2}A8y?Z`C}QEA^Ct8oVPR^Oi+>O!nBt99=>W5m15hZiyaC7o`ufK~f9 z?ZF!wk2&Op@<@$F2scJ9&iBc(V-QY-w44-S#6HuvG)cV=VCnWXkx!|g2p>A3-?_cY zw>nL*jAwLlh3)ri*($86fX@KUyl8Q}-`)y#aSveq=p~mN`~N%in_VHg0OTQawYSm- zIOZ4SS?K`c@dj?_mdoGrd8U9(!MIb+0oE7-@&`oKK%HV2RNI8W_9t5i8#(~nP4KFQniDcmMd=_UPblK$Jn zqQ}KQsrbl7F&Ij$RL4Hc@xv^vrI+;s`v+E;7v?*S66=W5I`RRS6JrHIEM5&R!Kd=` zbDH$y9DhwCM{F-UMdA;s;l*nSJf+ktX=e7s;#U$yP>w%A>p3}O+-=bLE6MdwNeVHE zgv7+C{0hN;ABg|sbl^q6TY^^3l>mn-khQXgTLa@OldEj(9K44umQzc+89X!~WI(PT zDCGznQrq)Q=6OUVI9^%0uGjbthXw}vA1gA9W3;nhCRUV^cHQcFYFs` zHPFQAq;L}gbcI<6_$5$u&hb#6X@q(*>FE1KCLT>r@h=%DEGzvYi*ek7Z*Pa{4C26aRfSn@qr|fW$FjNgXw$CC{;A~`+p94& z%nLq|z9-3{WpHF~+jS<6;2Bq=rz!6K!(JFsM;`s!H&9OQ{t=11jz;&Tn7iK>V$))X zzeotNF0;t6s6CX|7~g~fd)sE~%EDSRnz?Z$x82O44XzMWR&JNplvzaX)6X#CPK#Y* zKJqp0Yo)Vlq){@;f=10~np{yk<|t9zNkYngL+IXW@2l1BC7z{8E5!@ZX~$~k2RPE* zE_~On%Dv^R!gRcYDoNG`LPv37#>zJ$0E#9x0xLWFoYnW=0{FtoYeI;rYzwfe!!#{fRZf_qgw4P0F zOrj8HN=1q5C@R|>4`w=~3GWQCR^41)Zf^bf`ExqnzxT;?eqMFysCS6R*L3}K4Ek>- zKtSMkUw?MA(S_-345wr1%nkNI8kdJc1A}w82PhiAi(E~;Q4F0vVdls6!3HWsaoDZC zf?sb_DEHi%dqB%(M~kN(H>U!X;)FqU6c{lwPiL0&4@h>-yb|s+C@F@OshXHId)8cMGGKA;j&g-+-C}XJK{2@q2hlSAh|Tm! z>@1nH47wv;Y5Mw{PlplTnkB~lhIgAdS?Cev&VaM1$!2Ic&P%* zr|)y3A0NBk^nyKy`C~{{^OtD*nH9&`L{b}?zAlILnh*9$BoX=@`93C6ufBO**k z>Gb|!?~P0yK8_QDkuw68m_33s_7egaN4oin;l|ou?f2Acj zH+EEV(DU)|G|ueZd@27ix3I8lYPPd$I?2Vk&sPzOM98rwNLrlRmMA$>;OXnz389Ho zQu=|D&(=~{_qO?So#!o{m-%m(vhm%5{uRfm+fP2@gZ4B2BE0Xgb`YKtldyQX82^{C z7$FI7LoLMpdly~C5%j#`CR5E=*(B&zI<}!2ewyK!ao2AMAeuSbd7e6b58mFk#F8-f zbeh%-yHd{2irVw&*WpD4=QxMB82|(Se$ftvMBwG7V&`A$U>36BoxVx zI&W{4Iwx#xXoz*iK!Jfd1!Czh=Wfi%GQb0*o)5u5n%=N?=3GJ)TB5JNPr=&bi-QWw zwpNLKP0n}l!VbORJdeww!}f@mPE<_TL9vYc)(2Y;f*OBL5;`}}u)gwZ5Q(fAbd5|V zS7g{WC@vN!N~44YeNRL9)vym|Uqh3$kt;!QyzT4phRXL#l$tPx^=}MzLsh>PqvI5s z&N=JGi-_u5`cxGZd=!Lj)I)ypFE3~@s2E15a{X{RDpFm*!P-T)fQ0&FuBBkWe^HQI zo|yUK=NEfeDi!f6Og6NPg`1+U?)ah=%K0g}i1o_Xg81vhO z%g5`MOn1*jpaT{4XpU=;fkDgB&eEeSH3W4%-pnzu2>az|W$v^{M);gK$3-l1kt_y; zRdt5MpBpgiuxoNmLT~u6o>&AY0z&&Dovl+h4Jtn@0{_PFO6@JWvLUpiYG}p-T_5e< z-4Y*lz+xEQ)0MLN_R6~>P*yG}88$EZK+VAk_;SB55rLkNmO;6_Y8Cu%nCD9@zOi~YG%RtQWt}VxGgfyLkyE|NnQ?_u(p!!tM=e9Pl zM}1_@se=1eISXXm--v9dX2WYN^0kN7{Q;pP|S5U!*Oa9$Q>XopTS$&3%&8g+ zR%mTQkdkm(!l2xp)$P2Ph3Wf5w)~vB$CA)|#;S33+N!CaFJRp`EIS*&GJQI%i##Z3 z;1{$Ri%2&U&aCR>8RYzk-8h!qc|Wn}aT#wEp{6?dvqR*#-7)lpw*m%%78x1Yq(ChC z>6(Z+x4!;-X9|h}EL#ARgy^oJO2mfm>|-3$+jKCiE|i{R$~A1Dc!kU+;dPr3$n71&!BGIWY*Co5HVd3W;uZY%GFS?gWx}ti#;etBW^L*B#JW|2gDVH*`@mMxYkEZg){UZbJ?;BAL@ykGI+ zZqxv!*ZLaHZA#d;=V%!D^{QNkWVuZea4;hCK$3&Y;Cy&@978aNi`SW0SXf|NZ!?XV z;v?M$3|_Y;F5jXCA>sa+846Nn(C^DD)OTw~Cnw8gciBcEcMc};MiRFc>^}uwBx1!c zt`Oe|Bx`SWuRH5ca1n)&Hqgmhc;_3L@Ix|^Z%Nx<6Il`4XlfGsP_oa;5x^Z+I zWMUR!H^=2dl^~cFZEZERcE^!a;&xbMB9?F1nogQu?WcltwB|J#By5kaODHbjbrgTx zVpl?oI3KJf`&jVF0EnBMoxo*7)v>I#z8sflX znERltA)N!!Z&OPmlKQ#VrqU$I_h*7f=U!5L95oltP9Rr8!1wboq%kYR1KdN^Vcvnq46xf%Q2ZAHet4fZ!bw;r<+2hG zU0o9sRps>{Kkp0-3~as&B;@ItiRZHu&U`^T@Ti7T(O7nOZ9&9uod*AUoG1aZ7=c?+ zMF#}rHShf~R1rwM1L^ki&KL+gwx+7{-@F)4v-swS7atX4hxi41E4`d4y0i4!;IrJzQ4Zl9Oc zX&wPB27C23lK08!Cm+Q9{68PSq9&bpP z?IzLiy{^WR>yw6zy`OH1z9l!QEu5~pS}#le{d!WzcOO2KkFC5*0}*7`g+>MRHU_wn z2l#Mg)#3=)vU+`5-$kJLuSKZ!7(5xL?XGLI^z`Zt z4iBdpyv;k=-6AoWTq2!54<}52o5aZvkKr(6&yp!(qcPd&hL{B6E7nFfY4d+HA-U<- zt6*>DdjA)64YMGY{=mg@CBxCH?K4qB`r^}2KpR?{8JWk&va2CX584$`7Y@eS3nP^y+|_`0R|RqUjD+;%w}PB6Er zYx@lyt?)5-I1VhW2p3F|Sd7peJ;l_OtdEBf41KC_PaWs`gaP05mF=w&;NRZOdo|ES zfY7649C7ZwskWYMQ)LW)${A%=mz!9*(__!Q>wZns6N>oGvS4vP@AOX8a`a~uvV=sW zgaKY%E{i=GN4upzCzaI|Rnj}mZFjUxOoPlC9<@Gg0*XBzJoKF~naoA7tQI-DDIqxW z>zdDq6ySSMVO;KkaE+!xZQGgUX_tNN58#X%@pf`WUHknr!Z=r=Kdfl1fmGABPx=}{ zx2H>)!2QrPJ~C4FyGwYJh@8+huOX$c?|o3uLq7&TKmXJ15eXia82*|Lj*3ySK z7u%@qpXDYap-qtipwOnk7$Eprb?7YxN9qgWZU5rf+;;&PsWR^e>|Q;Wzgy;}Gpy+6 z!%9=nrzB^nS*Rsvuy6RwRk2^1!0f0LftIkO-lH`Lj-l%Ex*3c zfPVTTM@4&E-S-9@%x)y8WxkpNk<>cMzqx6cVV1vBaAytU{@F}iYK@^H!__~*HIW4@ zi3w3p^74q9z5R4>V0Mr6!f$korN-kn7{$5DW3*R(%J1Rf<~Es@l7iRjV&i)hrA9|b z7qm+3cE0&psgRmrUY3c0#{KDJVIn^Rdv&8&!N|7d=Jpi|*4_O`;(?WQ+;;VG9l$}> z)KimqpVtqH5qvOweQ_n>tJVXNd~{mm{r$9Y%4^H(^9=e0m0Y1sfP*6!V4oMV>U8xL zPzn5M*1%50E)mofEp_$kjW9Et<^${&ZP|qDy000?C+;(zc+w87PlhM z20AEB$2#fs+%{8w{M0lY=;^1!Oo%sUKB%3#alT1wI^8-uTx~u3)=O<+w|~MZG0@j> zK3&EpE5pxibST<^x_b4M_s(&G%Y6$}cX04`ZmEmWNH`#?sE6Yt#~AU(UNTzQqPjhc zDt%}#h-F^+ZB0jQC$HuV9~NAVMHZfqpE6&ghG0Pb;2}kHjAZ?;85m7TY(SUZy*(D( zKXwgl03pUFX*GJmD%x*3IAhyp_NBNQH|lsj3|{17?X{q9H~Z3e8$Fm*XK;qJ>SHDm1Cj7pyGcvaBfs9is4YvZ`V8nyw4XaU%gxV-9@2&;fH{L;8556xOf5b~hLPpM$koqG3+-uwVGH#aT z8|G5zX|?iuv6-&w$NXX7XzNkt(;k4ln>cy{kA&dy66N{}zJ|Xz%je9jD*2`-;YF) zo@0>uK>5obb$KP5-xtZew_l<@p<$s&@8LtgU$wY{SHfbCT}NIA2LvBZHVmd@k|Vq^ z8fr#{f^zS{)1Hl@^d6#&nUv&Y>-B2$Nv?4ZM4|o&5cP08M6KpIJ}nMB>P&#A)!IB8I?dwIkpIKAUI!Ka|2B9>5p&g?PDRoOdh!5TyF&>ss)kyK0 z`kBYt?+?SnpSPxKT~D?0QM-iH;yzrIK*3|ipBPBFQ%@kO9i^~q9w2w3`I2-xB?Z#f z8AR+5evl-LdLTO1tB*qYS47?C9|+*vDSj&Ar;#BRR)3oug_K|u;a_;c9-eFXC<=fu z7a$|XcSf!%YtEhMfqye?)RQuSRCrut6V7jW$6JRxbOIT<e)rn?QvT65(f|_~RTi z??vi6Gtkfqg3#Ui^CU>5R#tUPczcRijEU__9Y~QDKZF(t^^e8{V!*@8b#dizqykMD-hOW-?{HeT==|k= zpI-L7$ML1H@JhW^*0RX<#Q9EhAN{YgOvwhXogwJ&M>HcF1bV4CUGLxj=%kQRT*_+l zaU)EXS#rG!y4bxGow@%~Rnoyt4&|ozz|BWaKv2N3QNwIdA~VSlWT^7d0|PoQp;r5&N(8w1ZQ zDH%C-KkWBcvk^<(%(GAm`J^T1Fix<=PmDsWc_N3GCsk3 zqkslDL%fFSdj|&(@bNX3)qw9y+Yj*w*wkOBDhLY;D?lA;iDx{fne+2L>m2yLTl;-$ zo5T6fX^Xa^iL$zL8ynRm&S?zA0A~%rR`*gSqNBM*3;Lq>#8WGo?Ze-x!of=c5hwts zXw>suE$5@)mxXo+X$A}T^;XatRTX``Yey<+%5R6_vOd%S<;e{T6w=HV0X!4 z59}oIS`9z-pu|khcpo#wB?zvoaIPeJkVyIzn}+}vm@$&#O5A;7cL;(SLY%j?1#*`C z(gTgb4LZyoMZ$83+ADmmzXZI^jyX(DAj22^oRuYFy-P%Mus!N^J=c_}6LokSrZ}#n znf=Y@$h4@0fw%dBhIpjXs{A44e)wD2>yyjD-1bgQKAzgX9!}+RP4V%P_`DsfJXL9Ob{vA1{dpF}SC{uT zr>fnSfAY_maO!cf{+WGsD>`+Nhi8U;I!s7&`z);aOp1pZ5&{+#so{b~B2p{mkgIT_ z$ccCK+-_VMm2D|E^j56TRHQ*<3hdtQ`MQ+Iz%&sO1u?&))JoJ!d>_0ZMb0b%?br{y z^AFr{LXrweN&&!J;FrO~%hB1+l)dV4Bv?0tFpu7;Sn8n(p&kVld#Gt7KSc&(EA2eT zd`ikhhUR_m7jjuH3GDs*a0|xfA~x0m!xd_TlneL#0e6*tR&{o=YX~b++th$^j%jS? z8JR9l&ZT~(<)Ird=;Zu#AWQJLr7zv*=$D=t1<(jcZnuyu1_Tp)A%L=FnZa!iD?Pl>kpS6(nv1j{J6A>vdO{Gog6l;J$ zx`Q9T&1hn`_s)Nkw9a!RQBU7CK-k~+LVSzMM-2vce;I;UGd94k%M4%sS(cf8p~nOI zAdmvK22rsRvVog_6qHWbdWo=L5cL@ThMKTEcXUv%lt4n#$K1KV2n5IPFBWRmq-`Qt zmq|(OWJAPOimlutLbY^jvX08?#R-S#-i4TqlXidcC3e%6(~sL8+cq{THMQ!PzOV8U z%C2m!1(=zm{IVL)mHonTQ?I4XNw@a4wsl#zRBH6`-(_wcoOPd|OtR(j3UA*$d!wxU zq*hykou5A(KZZeCVS1#b*jh2@?$b=k;?%NCMGuX2urLyOvYT6X5+n2u)2C;%_g?5B zx0K)BiWdj(Wn#0oj;6=@Fu=YY0}*5z;AAxHIHI3vDz^qV?h?e^I@0_Z$8tvc*=OVJ zO)M+nweGqa<^35?)-Biz2wR;;6UNPa*8z{Vvf4T0yu>{qb4}us?J!-FkeG1i?!8zB z$?cEF*P?INoYQ71ef6}onEtRi)&wEg!63fARx;9n{)mcECl$Vdk3@0a96VKl zvRb$Rf-FE_$UK&2<(dM4T~by(AQtCI_vRQw)K_N|w}k=rBHg?$l_HuI8}lC)prC*I z(;RS!5aEi3paNC`NL&(v5+7Ys5ttH2C!EYqfK|}8PfG_oa_>_(uwZA3uW;Rr!$zgZ z-RyXGd{pUH!_^tTd%w6aWv_ga!nSM#olao!pp%rsYMVY^-??+2`=2P^VqqRSWJX5v z`Lko=DG01j5LBHy@?# zBNf%V%V<#phut-;rMtZ8VlPv6fB-NmwbIg5CQ>FsSr4I{biukwTm)Q_^ceBkV=KxVJV`P9syIb#~<-=l~=7iK>6Ot>t>TTtvkB=c`4D zE5)doaLYQgBDJ-c(lamIho3)d7txkoDlR30VXJ?3cU^QzSSufen@*i!q`F!ZcBkHM zoC*R!tJ5iAoeEs5FR~wsx0`7e?J1$YS)Mzhe|n#NFLcd8~orbP9%cY4Oj25Z&m-V>S&4imY-qMIRXy)@QJQjdI8m>!sb$ zkE)z778a^TO_3xs1^Uvav-@R*Aw#eu7l>S*iIf{MIeL<9*#X>nT=Q(^W))P$u>@ez z7B@b5T%0K>^RDHeaq>3Bnykg{xh*_v?hT5OS2f393L`1i>+DIkJm)V$t-pKptQI8n zd(C=e7Zpk>w3PiM%I(M?pzY$X@+T+6da2z24%dIoc*q8Mo1V71Xzb7 zg!3F^S$d}aTgdSEa14*}&!xbM3If0ZP1SH45ztG#(%1jNn#^nFGkbXE+E43r6c{Hg)V}<(RvkW9DwUmi z==lIOns}=_-YNEuARu8nv*WcDPP583`0-jttg2|gTXkUMLX}s_t~{;n+u}4upCM~7`|{O2hAV7# zVR_S)r?H<@pZ$D8$6`?9#b3ui0415!WiNiL)R}^|b~kWuoq2Vu;`K$xY~$(_EvR&| z%6=sc<5XueopcR9LVC{I6EbrOa&mJg_Lz7adNBH*Db_m9o?ZcRn}PoRMaC)z40L71 z;t!&M4fSPTA#0ghzLfB^xbSKZ1l&N2W~6x22(rCXjG| z!zFp;SPq( zpDA_S?_|TR?qnk=dv#odDN-xJ#DZXcP8w2CWRe@m9z5-ouo{aiNQeZZIM|}QDg&iB zrFoegXY$jeEotJNmZ>Qs9Zc0xrZYdi2yT$SIWD7y6i$;1mM<@eM);ceu+!zT(wD!M zBx{`VUlaN06pv&|c`pLMXw!AGBrOfii6iuC4`Wx}2s9MFHT7YRi|eP^?u5ME*hP=S zCso%NIvJszqx5mO!E0k~g0?U2h<>+G)p;c@i*rX(AqNayk}a$2WQG1pqj_ z#GYK%W@mdFLgj4j`q-C?8Jp9g*CrI~%LDUVnm>ct8KCdB(xCOer}1jvn|3#K{M0dW z%~y?kv&M4SYjfYxrB_~x1~&J-rbZ1PLV26&UF%P2%z&Wa`JW@lBeNaL)c%A?^a8d2 z(HRgOWK6xu=rX4#ZNCeLjI5eCMCX5iUjq4x1G-$yz^X$E8axjjv%SXUmbZ_IBu+10 z6tg^i(9TFKi;=VKNl}6*f`>dnaEJ~d@nGlX941Mh2p)6grbr$)Lr{{20#v+UTZ&jl zC3vVI&1;whh>oAUn5Hm#T5i8Ir+6VbIL(`YgQLHX#d30Tvn4+WmIE|93CZb3S_TU_ zMsMEaXqPlNGi57(`t(V88t6r4{xq-DdFTFp!ktI){FFS#6&%*WF`=RNW@BFD6SFbC zTO0D-&eXhZ*qz4b_}I%s;GYluo&rT>^Qy0Dq=3T9^iMVe!BoJu(;h+vT_Lsiun^MR$iw7A#2WwQHIAUC6 z+t?%uJ64BXKai6A8Clad-Z6<8{p^C7fh9@P#$ZY2nff6>JoyY+2zvEFB!VKq&)@&< z1FHT0?>jJEQz7gsZTZznd+nd&lH)<_*EmUsA90s1azdXH5tHsg7|kTF57Vz0m?5}- zH2;pA9s>?j_EJK;9#wL+PvtiqRgts*!bFTZ(Cd07pLc^{Gzr){Ot!5&%hkeOf%CeSLuy{g)EisCd9a)hkF4Mgt>8Z3KJ2$?759 zo0uWtS84Xc$%mjMwRRi2r2!BV&fIDe8qNL#wKZP!VnlKLY zx3-A&FeG>cPB20&Zt50XolIGUWP|t?z&j$Hs7YICKPY#(jnaq2o>sQfa5AFmxq^gj zFM$LLQh9~P#9hl1*(6201KCr5LAPq;VV`QgdEl;AtsI^R|r!_v1w9GPly4!an;Ql z~y5##lg>GNiOY8k~(jGQ?YvQpRw68Nkt=Pa>-bOCr~ z)^<1P@iOmGY8AFy#XEEYTD%UqSquAx&t`*cckbL}d(1|$Z1%IS&)gvC&rzfe7fmg;TgUybjT2*Y>>~MlUUw)h8oB@DO+l3po zr`E=7wHVPgk2rVn{`s^xIPa4nbk^v5D&cPi6P&SHQT}iUI40cMp6VJBJG~o{Bf=oYP5<$4AWy8Y$hK#-JrLxk{ zM+LIjs;TLZHIOqrwIe1?PYFX37%_ha+&_; zx=NCL3VuX1{k*Rh7$nzBp5dT`xb%u__o4xGv+w@~0GB`Gw&n+pi$JX{?g*(Eyku>t zO)U{0Hl}}z1Uc5ZSg}&)8WmL^^j37o$6i{qqa$fLZGA(&NV9v|KhDGf*9q?_dv&lO z!0r!C1Sjc(EK-X4{fVLZ!orpcR*HbwVlE5?y$j!V*uLlpr-iN-4%m`FYsLKkK^Fd< zw`X=>I(N}NCI0@6H*jnaAd6$m<0p;{ceoKsg;7sSwOdfh+cSt$_L`Y@td{?6wAi?E zWI#mk{^3pKu)BA1WoTbiJ5${)uH~U`Gh45~t>_;Kez-mST zL?-V9;G&ki^S>s1vLY@gUdex>R0Zq^)5Sz>hru@)&oT44{`8JH+EDV0fQ4|UkjA}D z5&6yG402y@ug)UMqIK^ii!C@_`;-|ss_^Ov+vlRtN3QDKLEq>Ww)Pz#i1Yq?3`IxY z)H`S45hH=&C=dx6Yx1xmZL{zoF~H87s5;tb=7x|))&N<>)GBVMbw5%1*@yKf`Y>My ztb%=3gu|Ses|wdD{O=c*)XYGDDqvYUZcL2&JZ`)oTkq6OiaF&h(wo$pJL_X6A)N*^ ziQ$LF6zAR{vB7`@?F0yIh4gv_w8!iaecvT|KDe-2&L-8ia729^1-plglnc5qKwym{ zTS5cGFQZKS6$=^{W9GyccX)30AaP3WAG?h+sFE6T!CGA!nMp59un~#kVJKHI48}8+Ar)NJ z9(SOU(z9Df38{LLJcvk3xDgTbrH#xMndFk*al(2*++GDmT83y$Ini_ber&08Rz$sp zu>n+ZLd6KGNJYwQZbE?yzS(R2;4$wkeWXH_Z3jAaES^1Pf7rKy$tkTz&IoKxTc1fG z-QGVB2&A-DseqMycE|FI5aGX;6Gw=2ubv*BS^at<$;c|0_~L^-dyO=Oqf#kk$@pP9 zi^s{MY5idtVSbSqVb!qcME1RuxQ43;buIz<4!^)cfOb*YtPpPEitYmbTKH8P8}{0|EtPoMD2ig;&l=!JAwh*}TRrv#6be5N8zpYU@J1YiVyP7+b{8K(&1 zX9hXg>XIDv_Qo-~FnLapFv}bV{|)Yuf)JC2A`o{6(;y(ZIk4kH8*=^ytFo$+O77E4 z$f-uY636i?A5t6~Rh7bTWo1e4g~=5EELsltn*y%P))Xs`U25hj5;q{S^V(_nUGDT^ zx8C_Vr%Y4e1YjVsTIpFThITWusM7#Zk<}NlaOaC;{g?t(M*xfltJ0yYO&m(i=20jf znFyDi%19KvvdJT(WsRFfoTBPxQ@$az!~Su{ID-_sSk;qBhB1DdO@N?InLsH>0YEpT zXE#3+IB=+>UBp%zCkU%cwb;md!KTh1N2Y23?a;7?Yl}seVi*hJM#-l69bt7t+%`4( zD#DA*ol!xJs@#~Oi4LWqVinMvHZ^jMjS2AP|mNOGUp&5G36I(Q9<`PjwRP9zL5hWG$C{qfZRMG5mP^w3b@C2Gx?Mhn$R7x5bZ%>H8DpG$yhCH3H&^N ze|XM@TSQ*5?33b)4ipkeNH5agF8_3@*lOn%!sR}396wVo{ogq6eJhIo4R7A7rkV%5 zx6zF!)_sKpn?vvM$I->JAul|muzV+%Ri@s-fU$(USUdyt$evYKcy^*rUsd*wNy5ED zs#ANI?d!2`Ok3sbuOeLL!bgoM%mFpBq7ld*o-u%LB#G5C2J*R*iGgX9g4~>h=ac}+ z{&q}wAC^S&SVYPGJNw1?h?rZ)C<_=DYfpFV1jckrKA(~-?Xdi^3y2zLeUD|UoIQ zB!;NP?r-M^^xzb`|2~a!JnYZ9*|}ilD{sPfrJfB|C#B zQg`GO(A7lY{5HC3Y5a~OrN5<11}a4SH?wYy9SZh}h66G~Eev9CX?p(5&i@cmRWny>a9K7tSRKURx;QYBJ>TB`y)1yt@j9w{I zdFjAL9Vah#ptKsp85FFxnRNxb zdGKl$epSl8Rn_PMNOWPL?%Tt5M~R}L9$`Q)?umK_dDLlD&hOt(N6P8=tRK%St*p59 z9eQ9@-oJ1RAECuQ>(r8ybFhjL9MLx&OAs-t`@Ft@Bizw5k^oxBl{sl4L%U1JZA%_ z{Z5n2;eu^$7W9=ofOb#nctanche_k=3=El>c2v~TxIF{P-_7aLF4=O!>+@+>{q#|1 z?Cu_!B=og9BHo?+oJ>1KNmYT=9KR5dd+eGm-}H7q*y(YDV3_*119P_%RLcFokJ|{@ znp-iNwQ!byH(`r+nyauKIa@LQHlIVBJkT64wKD$JvWRjSJ=|N+RVgReZu=G1d~wkC zZ5W)kcX-%>y()ey5cwAgcaXcvrYn;ydYX5xw7k49HRt!B_c2$FaH3$bWa6m?tzDLPcyU?8UV`0Z=$6C9 zDavd#H+$&;2_>eniQN4TY-a*q$I=@KA8QvfFU(FdlUIakG0M~#t*p%EdqHKo)b%G@ z#(jKL+|`r_2{jE_CaVH)v!yDmj6kafhFsy1s>=e1y)#%)soD1gSkbH3ugUE|7Z%mk zLCa-TvlW)qdg{3d>jR8814lhJ0h&Zj%1Y(7KvQsCU)XJ3KCl3yF*UzOa3TSX=9lB4PDl)y$Sc4 z`?4Y1yS-vEkRz}lKz}$0)OhRkD*qG=)P6rI&hFVS80XiYPD3hL&O*G-!pb{EBDz9< z?5<~@l4@@M&^ykXR(t=HZbPRA$>ZzR}cV+o2tz03+jIH_pJrg z+*uS_1l!Zkon9!eC+Ev12Yjvj@@UvK_nqOZ+*0#u@<^w%#tisGUC>&}iN9|R2N?sC z`GQbtdM49me5Qqq4qI|35;>CNb={gtqI2SRYm0L}W;lDHt4%{H9r=QM@wtrTDMgAM0~`V*_e5$;gI3gmv}wo$X-=m7=phC(M-U zWq&w`%u+A}8C)D|K7rv>5xp$LJ8LT6-PL_UCX{m>Pkp8NuO-6CdFYIAA496p3MFZ1 z!mjhKM{I%OAR#L)mIV&mpz4y$i5iLL>cLIQ+_g?eOCYURxko#bD)=0i_hR&GeO|`O z-2TeJ2?SvF)1?)Bu_@gga;>dr+A6KMmraGN9fqYZc=Uqqgj$U2V*LXH+~#8aD_ZY`H5sog#7JTgr6ZpU-}_dGTeTzlvVhqj1t>LxODb*#x1 z0gWJ_17$3rd3ZT)Ts-tm0mcL;kTtl0Pz>)ECNly*SbI}FwN`R2`9#q+ynla)UjWp< zwv3k)lQ)v_BEJk)I$|>#qUEKN7)WG;NhZQH_RSEF0}=!_jm5G3TU$#?gg~+C;f8d@ zbq=vg;dt(CNt;_eHAXQD9Hzu^2=`CApDgnrk{dgHT?O3!kMs<}HI<2F#x~fvovC@~ zU*zR4>CNmKXoMv)JO1f(5pIeTwYxG`(3WMrg}@0_6-hTdeP9{L6`4$a* ziQOQLZSR-t|9v21dt;Yi3*D&R_5sadm>X}hl3nQ*f2sREq#^pB2{lW6GWdZal^XJ-ei`}Pzp zx0y$J3)yU91ni>j;~`sN=*7ioP**ZLs~0|gY|cuOwB)r>rLFm~M{}E{?;nm5Ol1|Y;bWC_`v}#UqH5|AP z^9=J5 zf&1RFV5Eflg^r0q?uaedY9tZjlzEIhZm^h`+w3`U6+9qEE5&96mmJ5Mr?fn$O=4>S zNcpueuhehsgSFc$0Ooo>cCQ^Dr!R6$SKqLcHSa8~BC+?fK z6ACVe<`_oW;{7)R&uAaTdlRJ=muJiJ_lxn9>FDQ~y$%*m5_+hIA&Cx$@w@0S+`RE+ zhMMM{uFrba8T7xH1?N3&1=tMdw+7H2C+T$pN1{z8$wC@yQ3_C@0e@rKqZoOkh5_}T z?r_h(*;=i{H>4Dy=%bxj2 zuv4O++S}+F4v7B8B%JuYg2<~1CL#a4u<-Gvt=x6>YdiM`V|n-fn+@woed?ZWT@K!{ zSBMJJmLQ`@(-brWlC`oL!x!lNxD2J>`p;|x7x)i;WX)1x?Ewe;Hi5e`{3QE#G2Q5-vlYLC%3Gg1@IDiA!g8%uLVfBS!L45La7G`EN{ zwv^-zqAJ^F{frjm5)JR**g++cqo~5}ydMimoah*lFB!7_H)ObQ^l+edzxUp0Nm;ZeSvuY-$1MG9)$?=dEWYcrxf&a z#kG}=%>S@Wj91~B`>~8!5dVM+4>%*SjU>P^qQj3kDLWS|UW`m(54_qAr@j%B9lkMXFk|ball=E{T3$l-qMd;V~F`*KOj}jPE|Itu^+E(Vd-?~yq{tpWuyqi<>*Y8hJ zV>)k*g%$lSNiO+va)r^#)-3h`dvBw{eI2FP(8DuOofqOAW^fzA+%wGq;11GXa6DVnJlyI8YQ&6 zZ6rNlh~UieLY;r(Or^9T3#UqngE6>GXY5t6&6m;pg7Bo$vR9FwV>a%>i8Qe9WLZ(& zo$@806DfXSsyyvz&4)}E5Ob6Qb`h?2a0;2dSKSG7WAY_=-)}h&4L*OYSs1Yvn)tO8 z)lYWN8L$1cF_Pe=xA*ykV^GX{7kOPOY)Wjy<;?6`b4Ty+uy4tVFOA`s77VR^v0)bWy?1^bnHiUq`&ZNfsLEb z(RTT$t&HEXg<%)boX7br;mFsQ=FjJnvzu9Z2^qz6uXD_=u0G84eO_#HxGt4-eOF#F zK<_2n=9P2ZMnOsb)*@@V)@nNV!nj%U&8uC1LK(<+)$GP=TZ7cAfdYo6GtHI@+p`~x zEte_(O!PAhMIxW|J~w0dTw;UM3C|O3CUs)rnJh&}-#e~cv_&B< zn!?}xA#8;3j-Wut%JCH$+1&cz>dtZLWsv1rbmq9-fqI^1)#(m;fX?_O&QyM?7wa+Y zi20HkwH;KPz1RBJl&I76>(@M7mKE6f9Y&G`52nhq%kyJH2dkOa^qnT;njbr|3%5-( zt z4M&zV8B#9p9p8dMmz|m!Q%%0*5GBH^@2*d?H;3}aE#j?!yt2l=OSa&tnxL*4R`-N_ z6_*bxnp)DFgS#3BbM@e5 z(&(P(sI6bYKfqAVRtb!!PpUdGGCw}#CpiP9K#{bih`PgGsZ99O;b(GL`Fy-cAZx{iH+_aIztP3iWPH>?N$?;or$ z7MOMo!TU zQB@%N22V@}Zg^Lww3O87#@#rp&B^1LNx@Opuz9rVcx1nl zjeXBf0@DkC8aABi$jV~=ResH9{=Cp@zO}|bFK-E0wdWtnRj(F{3!ya!retR8owQi&06o!G;}Ke^?*0>yEiA&T)X%9mk@t`us&uw6%4L2YovCzf zI?=Gpnu!uKPOpJdU*ip6< zNUwPN)~(#`eP-sL>E%TD(aA;`=+ zEy9G?IWIqdmN-8-1L;whw+$ECd1(@A`est{UbMGkNNyRD}Ja=onWi|1(XpUdbZ_oN-Q?r^5D9^vvGKNF1-PwCRkY zXFi+ckls2hm@QJz9tjeTQLI4KKSk=F^Ys(+4^S>RQVW8-5#p>0MCs?L+5Ib2H*t7$lo z1{hWX`;*T&@9BG5RK{&QF5n3=`E3@ilWm$BlSgmg7K}Ulyu=_OBc(g@vHgeF6U$?tSFB!^0$opYQ+upr$)Eoi8yq5|5Wo>XASBpFqVr{;ozt zc5DycK=HN_S7Ys0uJQ)1d0C+(l(>y#b=uv(a6A*p$1`Pgr+nX@bUXD6{leTerZCVD zyr<&e%XWx&fUj*?_*$22T#sygNQ9$RlneEp?|TS_h@_N}i--HmlA|X*nZRYth zKlhCpek^%(VB71N!&b5HAaoHu7po(i#Qq1nzz-%5dKjdAoO>t2hxifVlLk!>?1mVn zUHP6C!mO{g^4WJ?TwE67SXO&kQ+MdxtYrEt>i4DVvJCfZwGPx}IP~(SvwT=Bia?+fu$XiL(q@ zc;fw)%(Y&Tm1W0obWu|iD+=q@ItcASC`{E z?nO*bt5IN!^5p~`1_lPCkY>mDxEZ6@mW9zP0Gm`1vDcUO?p^^(gk08sm@cwKUFPK# z9BSx8gi%_i+V=Xb9c-~Tm4l!@De2(&G@r(zq6V*)=k)srEqe446`eJjj*g(?tV{3` zN$|^hV`7UsGJQvlqK{km?%W9uS%-S`^mKh|;FRRNeUp&^kFhq-TLyBIgsio4l>#b$ zv;hs>)P=u2+MFpa^PYNI3*^-34$h`neestDIq5fYZ#5dOY)#I`(33}6?*y7&tjRun ziFs1Ft+9ah#|66OWK8v6zEo++>APOL{-G&r_@ZT*p%>diP%IkhY8y;z|7G-T5$F@e zBRcB46(AiGk8OLyWS&{Ufo$yF;dFyX{>T2RHL#O%gc<+Jx0fTQZ|#j6v$LTVg+C>) z*A_lzl6;o%+LrOzv_Pug;rtS|+=nO$^~geuWCM-E6tFiuvSDGQuvaj}GJsA{&hsOj zT9~NO14&6_lSkrE5p~!o%1ez(%3UZsL&(GP*|RqeChDG^Ax+VtK_O;tOH%d5rur#n zy1LJ9rUnORb4?iCxpeg(7O;={6t?^s--^)w#1*qRuAf8dj`w{b7HAlJ0kNv(M_ICTfxudWhj*z|NF$CMbE3#rii= z`RW4%jpBSAF&xNn7(f|b7pN=(~+P*5qGQ7p@Y6&m*YPkbm7r&;oV3cjf#xk^*(+Z=5 z6&cO)U)R>`8NVL`UpF)o3qq{B-tL6>4H5WvuJO_!EVL|v=qeOL=ukgsrF-uz(ce(E zdPUB%uf{?=-?G!x1&Y2!${gWp%S`H*FR|XkttunyP~G}dqZvDo8JLGZDZK_lh=LV^ zXz@1`J%wxbXQ>+-9E^1bq#9-tka#R)oUdh-p*dgg z09ie)tjYZN&<-Z_Ydl+3HZ##<`)6xYVxY=>UIC65?{t`w@tubM($v-Eg2uO0vHr7L zt4wl-OoBwoT`xU0))^&PR;6IOx>+-qn$L^&%0K&KWOgR}jh|T?U%Pni7+`^N8h&r; zQE@8{`Mky$2U0-bIczjnZ^rj9W~Zi(t|Qz;u_|rYSK*|4r@unN-*WVL9d~jx-bp2F zhvZp{b&*)D<+`4g2ICXN6dU)P0{iKnXcNAavR#nH7O$d2<`HtQ{r0_wvOdc>OIh^H z%)z3kOivwW69jG8gO9Vs<9(Sf0#%07a7Rbvf8#xypOQMG1IZsAlcqtSXGL$x$w%tmHS{ig~7VPVSikA^CR z5fr+3Jj=F1H2+{Y+c^jOEWq59K6~~>fY12#XFVN*uWVZs9GrtHU8!f_bh}kaXo~>B z4pA@I#?T(V^5&){@K!0>Hs|orBStUxUvTEV$k)a%&5fhnX^Wm3N(Qf7J|sZ|d4q(J zesePS{x;;O4WMC>S+jG zYv*=qI~MIkO+~5CkyQqj_bEq4Cz1m`@b{wI);+<1sQDOt!sT>w=CH$9^!Fdl-TKx2ohw`pEBulByumj04 zP^zZ6c~qNXCTE-i%qQuP6d;g#h6%y@uPfIx+7IhogF>s`kE6UnczUCcR>?tX($q`m zeMPdw@LV}KgPq{LVEz8*{Hx*C4F5--hx5as;Fm66GA=T#j}kFWGA5;|SU@P+g%r@- zw4YC5hp$w3oFolEfT=ZpYtOR%9}@T8GpIv_`%#-^ABmDMxx8@jf-Q&0r8+L9j7V91 zn{=dyU9aT)^PAe)F??C&|6avI9nFB(=9hfbw!dKt;+MBCJ1fi%V-|=FO2KChoS?tK zBmaBe*F-Y%Dp?dplP1X&ASrL!i#YQy_?Z3o_d&G`0>0eE;pPegQ|%gYQ^%ViBvCH- zU-bbF&dgWh+Ztb5t0?R_-|_$7|BORqVf68N_{wl_U_@{bvdkru1kcs&MlE9)-+Vh1 zaJEHRVrj|gK<3Zb`{KV5y{~C4jN{TZOPlP-tQ`j9Ccc0F+s?6(lS87cv9apgx3*7^ z9+e0z_FD}7q?w_%f}tGP6-}JaPA^_pTurWf_W=zt=?EgPkq(!pyK|Zswtw zFKU%yyS%09g0)5FayM~*SexeE`N`u#cSaeo-z((_K*`{LF^B)(UcPxv4*93JB#;@< z6~Iij<%Uq*K6Ao`kPA`MP%uzYfk)%b-@3Sj13H@;+P$N_hY#@f_Vu!U>=sQ;~7t?oT|?SdO#-`ofp&1Av%YH2xm85d+igOv*0w~0w; zV!X|72K(=`J-k6tIMB~uMw~bF<+MO$#s8FC^zma4t&+yIKA^iOz)`^suh@HNie=QZ zJS}#bKh|8=*8G;+t}*}^XO2UoS1?hyQ8NT1v2b*LL@Qwb3kg&?&h$;CZH!*?NB|0* zQO7}=uEg<~G67xdwJiI&3Rldk@Qlgx)>cB9BfN)K$7ed@v#x~P8H4(;QMgg1J9_L8 z!lp(m?n9zSTmpk#UKS4wWUDTTP8Zr`uUD2Wz<3X|rmu`@U1kNE+qaITD2Rv-rem{@ z>G`Vcj%9?f1B~+w(v7hbDW0doA48&eIJiz0Fvpk&sWP`i)H7xDN)8q!0E&v+>;TgR zs-b%D{_ioH|Lqx^XPW}1lr9$J^`XP*Rd;xuc9_~xLvPSE(Exei7tn_6$wn40P|#UG~SG zmQ<6cmBsbabP`}GZW`R+5DyjQ`cK*M@%~wJuMM?-K?nbC*aP09UU8jWElt=Sd9s)n z$k^iD^6guc<=FwFgzvUGT%*=)I;1`HXi!c|&Av0s|E^_*CNC#9GY*e7M`9JTvZ3%8 zz{r)3{d3T)m&h3Yw0TSxq0TQYE%~fXYyP+}V`fN^m-k* z3`z)S|FG@5R{#Ee&fMkQY{_C?y_2(3GnYOGfmC~J$m{emUNnOoKDnTknXKP(kjb>@ zhC3>5l7M&!|N0^(oicBCxAAgYDL-!!U;*8t#@^iN?&-D`1wT2R@f^x`?o1#Z%$HMMRtXK4`Z5MU#h!m)GJ~Fh z936|+sPkj<%sLcH#J-_e8*@-Ie06UZnpPac*Q zCHz0)-a4SE_x=C35ETIx0Ra&e5d`UyoJa{sNlbDgA|)W*8;XGB6r>xZOS(ZoKyq|< zZ^Vd=7_r~YD?ab{=lA{o|Gob~aE!Ba?sJ_h9@q2QcBNER9rN{*-=@>Op8~s(WRR1U z#d`YLdRoj&*)mW6rH?r}I(nWn3_fQw!V{*Q&3(1Uf|L^uGKh$|@1x$nAqOWN&K>Om zCU&kbUWnJ0{{wHxJ7NbDZ*f(zOlZOlLpI-OJ6+_={IHs(rrt|vR@t^I&Xs&rZ-dg# zF<`HFT^JviyCYK{5*Vo^tSdP?@9TD)X>H=_|5Mu2irk7Wy15SrsH-<`L0JyQmpU*xeE|BC`(06OH z`gL>Df593%Mns-`FQZ;e`%RbEGWQWk08ZR{AaI0X*ueDMYd2SophmO+Bg1t|(23o- zly~Z*#CQogxwkJe6`vsmPCR$8t|xADYEg;UL$CeetXUd;<$Z~vy#LhO5OJ^{IV8hr zHV2;q$c69&*LBtWSEHVT`UJan4uiP3IK*YS5`b5$`T0RvG&dLegdRMIEBP90jrcm` zrxO39#s*b35f?NTLl10`e1b<;TJX_HeVKV~uiu7RQQTk4!owGT^FRS@7<;j|*EW|tQZg+eHePD{ zOXe#|${qHhtIQ07PTM1w`2nQ--2Mvk3&^kMX%{afCicBVYC*Ly4r3J8yG?U0al($q zv%>s@^7{*3L(DPMfHAZ>mA7URDcRdgtbbK(alJM=Y9jb_-~F?m_Lt;;QhcrU$4iRY zkAg{Q9@o!W`BoQWJEjsRBz*Um%K>*~<9zn&vc%rDd{{Jy<`ogEjsNB09dE_=zNDHj9gt)}H zbDt^}4vMp?o!&3Z5rt01@9|ZEowUQ`c~6YFwklmnNb=s@yFYVFTn;1feTkWM`MbkR zlC>sN(w+oz1r>Bcykp3a5I}w$MAuvJ9kK2Tz5xM!xt+y;6BeGZ!ye%J>GK}u8^27= zVGKNxkJ%{sHy1$I+Bi(ObmelaJ3bHMxu+m4?TPVklJ-T$+7IS=9#VFoP?yP&D-)%( z5+$|OWjDMcmZz}XkN<7x0{6UOrL*i!XY0I0GWIJz@@7%C24UOVePorR(b4FVm-k=2 zOfcAQ-CGP(yb)II^kS`%CrCR-qyjOV?%2V>(eZ@?q?O?R@(rvX@V-YiYo$3f>4_q; zG1J`B2+_m#*wD~w7H4$wpv4|`vdn&*K+5FqQ<0xvMnFy1RnBj;f_Dy%=kWvpA~Aoz zx}xI4?-hTCdEa$QJ9~a8_hD?pi~slw;M_NzigK1Ufd^ZbW_WnS&)g7!vw7_o{CvYB zBP;|g>}=Z>wtxMa+1}=}+7T~RTwGtp!d~iU_FG(jYJ6~T5FSoLRc^#G#cZ|VUkH8v zN=8{(;`9#i5>m;rx_f$cKbWVU4{i22NZ0FL{?4OftyGI*0Z+(g7QsO+T%Z75M^Up6 z$q$wuQPMo8ukSRVQ0e+hr@!~ls!(0k{eI`)@4(k2>Sw355(@3e|GDeE(yQ5+gMqPq zI{xYY)qh|6^l(VA_00GP{lDE$;QGwnqD$T0+uVk-eYXSHP*}B9*YZ+&f01agibg?R zUeAD5rVgWI^<~a8(^6J9L2U8wx*PtYVc&;c>*F2W_z1Y*ioknM1_pH#Hq{nZ{aL2u zTn;FdW~kr$?ipH=I`1(75On;zp56#glz&1n zFGr0E5C+(!G76AU_;-2P!)LR9Jt?^8SgW1GvQ8pF<0kX5&=9wB>8X9AvznL}$QKF- zXYrr#l+*nEJ4u7vS8zBmIXW+Iuu!zGu7^3LRTkMT_9yI&0Hp_MWju$mzcqui3^;fH zuG{JawsR@9V>sD>xDMNl|n|Ish|X>gX+Id$1g5!1m!vl4^3xprXbGcve)5^ zSf+&3e;wA^_La?^6|U&n*YR?TexI&XQ2ifo;iEWJHw?d)B9EEnWIeOtjZNB>k^1*z zffV2CsFs2+*KVR|!*4E8&1f0E3bz%n&(EGJL&CnO>PLQ{v_ZxVd}{3E>to~ddCbTA zfbU+(gZlf24o1?B=go7|Z*zO)rPrjcFjN~O|IDxlqVJuno12h8#cmjUV&7K%TkojI zXsobRXdgse{K%UUlAD_uNBniH`Gy_#aQp!8~YyGMRepl)EKDSgmt*2OddL9?1%2VDR;hrd}H6Uy@ zA!}-C`MVPl_(5nc5Q=r)_|XQG&@egN{k646+Z|#~7?&n;VGlxED;y5DsXY$y<}@`m zt;fxgQHbh0;swcQ9;>m3Pe^KLX=!C>XgF_7*sk>4yH7WnCbkhZTj(GlvOajQkQNC`HkDk*q06V?fq!lAU(JHA z-O3yI-L4%cQoonAReqX@j4ZHdIZ{7ZR^p1|Kj{^Ctq6r*g{Jnh-`wW%o;Q`?v6v<5 z@4G~FO2a!W`@I{FzZglq_wj4gShStAr>b{zP-r-Kd9Fw3k!C>79VVs`DPl`Ldx3hk zxcK)w{+@5%O^D55%Y&IUH{5or7sa-2d<4cvzl!Fu%bYKR&4NwemKj0NY7FN*<$5>_ zgU^OusrfM67Vmpha3nD1uA9#7ydT z01}xhjcW8vfA}Km!=A)m4RNX(a?p1%1N+lYfAJ`DZYrIj9ol2EQ zZchS5o|ue8xZK0AUdfU#eUnM@dv_=$_n+E#Qt8$k9dCYP=HOtT)cci0z@c~AG7?EH ztRTzt^Cd$Pe(&$Yj}NCEmuqr6ALk?W{=W3`TtIk#PO0R)b6AAaeznYWt+XFS>U!z0 zq5HJN-bhYQMYeIqg(}qU27YH}@f_iy>p?h_tKNf#*TcCdVyI=xZFT+Z`HOW&eSsc` z9IRBkM%EF~2ynw=Wb0uGV5rI8&uUr&)b-0Hc{%ySqh#4d+UzO{|9S%2pYU-kIe!0B zdlGumf0iE)gVcSeF|hXZbdzbFv2GI0qZrdN(B;_jueusCfARZXfk^3g$9iq+LzpDu zCZoYV}i*6~6rl0kr6cJdr_xqdbsd5Dd)xH%BlRNTv+`b5SLk(Sn6PCS6 zDJ8e|=MQMNfz3FmC)UcNMwEr8j>aqtbRDhR{T&tCqn9UL=7URO?6w!$dejIOpZ%>r zw1qb7>+tfiTQq*|c-I{l9T(@(&skoBUt;*p2tUL}Dhp}U`66%GPC5?v)9V?1+=}^G zKH?QpQrtBTY0X%&qZ>25~_D zxQD1rlrBX5X2tLeMUx2;ZM{PHOcS}15Q6=X7WW_(eSDY7+(z@S^XH}KZiqUsTr<#p zSz)vH?p%+!$GO^ynVFXU+z6{o>nCOS+nkv_*+2T^b@cKJ>?;<#u=&{EN*7Mkw2uO? zw}|`aJP&49dYSj8+zaGV?-2sun5>Ug?)Whweh7x=;-&B~8iS8i@FH#`2^2DQgRo%zeyJc^QI%J;sx3^EZ96NGJtNsax8qiTw%K}+6u1ts3 zp;~9`6fT^TjqSNac$MX_%E`GmCHXuSxLfqOuj*d7^k!G=SDZa2eejT_x3BS|=b`j)M0OI(JOx6Wp==Xb0|E+#GSr zb9k!1*S>L`$4c00VkA!wT6O+BDa8XWOn*7{YlC!lPENVixZB2rh-BisyK4%lmL{a6 zqPIy$#Nq-y_pa!h&nH^1s;xN@oCTDUE|4_)_CMcJ@3`$>cP9xypcLmZ-@Zw>2t=N@ zfZY8*Wx>nK6wAJ{q%BSqrZ@StQ^^7;K^Pl@dSY=j&oNNKC(})LY?>WPXm${OVbMEH zP0e=T`uuqb{BcFmkE)`In2Y>ahPFKa_4SMt>Apdh(0qf(>d)Sk|Ng!)8g0rzj}Z@N zKiBiVo3U##o0+j%{w5vfHlw>-A8^e+=cHc~21W}1kCD=S&dDI_~yk^cI!;noBziIbR zqbBiiMfi0*?|e(I5C=yXMXP?|N40Xdc{#mjM0i97$gRw9Q9ZI|7 zj_YmDR($hnX`yl%XaA*wKtjMn+&stI`PFQuL_4ZSz_3l)v3dcd2N&B zwTVjOhIwRUc=+`;(}bOUDpjCSPdquaX+>6U(uMWCJf>P)taPd^0EYB?-EyT%1C{Dy zgQ|s5(=|iSK$?`$8(}MZGm(jiZj%)brNFlWBgO9JKTUPV=ycT-6chmd@-u##-+1YT zTb0@7uAa?Fx$Eh~2@np_6gdxW{g&hfeksz!UU?&WPNf!S+<+!6*4Seps%%}xEq%gl z>-uE88JlG8X6jlW;SYG{;a!sk`DXLA?x(szGSS=Y*8*4XBO^N2?YU}nVmJ4nE|T!I zTpW(S70ncWR;s~a#p{efET(VFDv%NxHR+CtZ>pkPl_7gDYukV?C69#*7Er842-jEF zQR(?C!CDOE&Ya~SQs%mnrYPdFIYUam8c_hNKl$xHufp`K zX{8<{iI)|OY|8lf3=GuclZesrPf<66nyK=6~o$*R35;;-{1G20D( zO!}UKS}5^j2OaZKplfPfxu~7*{hMEv90+aBnNXn<5Zk@2GBH5@lfk;ecIMYdM@KwN z&$_D+ex3kxp2S-Y<)#;O&^2AziG)e6R;}h!t*@cc3LJ6*uPQc~;BHqu87l+S9UOvw z?#xJeSKJkPIAkFPpjgCKbL5v~@!fHo`oBqgGZCL?6G(kUQC)fi_ejQ0(i7Ei`ICb= z7!2f8`QiD!!oT)MO9}M&l~sZpDO)7GsM!UYG|Pdqy)9!|WD+u}>G1SUAj|7U3`mI=<6P7m|hhwKXr~M-- zPV16N*UN7y5Tf}P-dv`nOmrE2K!01*?)ei|xq$?UYV=p<=g2bpE7m>J zHb6>HknpVk*(C6K-#FT6sH}ZWCN$3m1_aa{6D;X_=sVFM8=>G|7RG-YXrzU@ZfCKY zZ#Eoap_-yP8p~N331AYP-`k)Ui)q9~^hjYUO5$)Ln%`We=nT;-Bf-tCd3#ad~y6#hFIoT3i0t3m8I|I6g=ko;{2Le*QSx)ognBw+Lty_?7R*%ZzH)T zY71>`TQBUqzeqvB8TJ=s;^p*!H_QE*op9@EQGx5H7@~YaY3xos0?>cYknkPYMTj}!m93LJB4Hi-5yD8Vd_ivUr zKRH=NfwEUhW>-KqeMUFT*_O7#JcG6Nv+1sufi^x8a6loSHqK z{0a(~CLh<*4c&|Tg6=KOF!x4W7CNEj%*@nm|NgzZQHObTC|?lL+Z_*#6&WSf^jaWy zt+>;~lTzKe-^l|orzW%+}AvnD=n^Q!JtH-)L4Vt=IB-Lt&^BGUv!OXqU3m)Rq8Z7&d|}* z`zJr=?B*7jS>f13J}ky*cp5R4P?KW_7>r`8rvp9bSu^E_Q=-3xd&E`;e3%kpX<7Xj zJ|9l)p{dm`CriJ>z1j&r+4-QA*5bNrT(FvmsC&B?KxGJTK{i3d2NgVM;^m-vTFZ+$ z0V_9eOml7{hlevQ2NQge>#OA&Q#FYF^8$$r@LnLIjv5|Lm|Jg&cPRch+;% zBn*V4ataFmKVu^!hk;D;P^sl`QD0H@QrG;DV`N>yI{TG>NsaFpJRW=O4=#SB2(|*N~epN`VajVlfg^Fsf_g)jo zjL~rF3Yd1sZC(7mdC{=U0bNjk(g?E(nxE+emD~x>f|F10Y-$f4egpGC42kQ*u*z3y zFD3Sl$Q<@0L1fIa)Hw@$Zo)YoWm6I3sB3aeq2J@);zg(24y{VwzSPk=INqDqU6-@4 zz)U(%9&B?IJY;31WE2g$MUzRKD@Ij(QY4HzPSb8LW#l{2o6a&apX(84%6~oKe)7a+=B{VM z>*0LHvPez4%Z!1<#bub88Vu%)8|kHLMfQ)KM_`ki{1E%Iq(;O(zMT?ZX;Mlsfs31i!}Z~FGS(A_t{6cF0ocg$ z$~WfSJMg2CLZH5S)Q^(PXj){J3(_xp{|FJ8k+JN@W!*?r;`Ajf#=Bq zZGH`gT{n89&g!txxOL^CP36JTNW~L?a6I;lY*$KF+vL9mMtN2w>A$!0H{P2>5WxT# z8A}Fkkk!K=SZd|USce)8KUyF7EGHgr9b#FPqXD{&fS%XA?tyNml_|GzUou6-FwQVN z4Wxl~#w*oVSWA7BDv6i$ltU&+K!`XE_nkxx1~Y{qO6aO>?e5;;wfoIfzgnQVgd^Fe z?W9cxRoF_ep@`21Jv}|Zm-%)PNV$itIN$bEUhYhgt>>k>aicpCqT2CV`f4~eVe)%) z^rJN$4UL_?KB8#smsfr#JqKAk;Pl$Rxe#i`jzpy5|}JcS>52Jt`tQGam!%!6PU@$7uCZn^QS z(X0WNC+oStLTgcLUFX@tRwl$rrKykeq$f%Oq@WCdOSivetkCa)AB;RQJ+eakxjAoq z*xVeZZgyACJ4r@|hc}bpW<6eVMG~YWuHA4Br+Ek35LCSPATCJA z_+*tM=q43$!m(L_ewfg(u!A`@Xn)b|5}d5Q6fCtGTjw_?-N8(s%vP=c zS8DYn(veFX7H7NttbkWxg3U_GcdOr+rIPxu@I=-L!zhd8XJmT$b%3V6=rZXG)jaTQH{`5^Boi&o2&+muS0Czb%X|B)t{;9HLWD3Z0=ASr+ z!wXkOi|vbwX}NS?X}(&?(wuUb?}8en)pPY1Eh_mGD||7E_Uq|UGfGdEvJ$Lru3fp) z*FXq~7nPHf#n|LqWAJtQ)Qr>;Zk=WuqDt;g*ktdMki-6aSm5-D^4{AD5!*OgVmR)b z!+q~YY)ZX}GA^_l*Uo2X;sF-DTc@>IjvA2jw}c?m2dIB z7=BgkMvl0v>8M;fh_7(TPZ;sL<6-var;)3xE7=`H5Y1Kg_EU8QDKhqEFb4W6!y{Ap z&XRJO-!6_HwZFfgt5fXOfzYc^vKvid$*J=^oU%uGPIQ3+#0_(^=!L;yScYRd7&YMb zvwSk0d-t{&{Dn&mZ=vmkL$po(u`VtR3+9u=J5R~N3iNF14hFI`aEl3ij_p4b!q5gV z_?Ay?M*4I&@Z^t3nLKaaxpT6uR3N!OOweR%kxlYiFE$D^BhiBfPw-|2-x&co1nvCvpozC>+HQG7t%;jxH*QPB^=uNaNr zlPf3AUwVcezbjH7?+Dd+pEQ}0&=y_xs$Ms=eFMVey0cTqT&vH3{jN+ESw_+o#BBWI zgIquLWLYrLrU{qH$(VRvgQwr}v@`M8;eiC-{ZNJ&$WH_jNLO18B2Laz_HY~dhGW3FpzouH#NxH zfrKr+h~0pJ^Q43u?v^ei8U#*;)`m}CC)N4^ZW{6gagvef*$ZBlH(r(j){h@Ok##jG zNB24+M+!$OMobe0Rcxx7pSoakJGeoWC6X=1?v3;Ml;i4pdH=j_mBZ*qZ&EX~+3zpk z1nM_!=0D$hWO`EaaWI0(ea#loKv;pKHr7zqx>C_Sjm&{Q$+ZIcWO94ifTS z_>HQCg#{6QEKDpt44s#{n=)UUI05zq&N_Dqbi z;&**RF7?MAEC&)jj15c*EfYt=Ku`4gD4QNnMYp^~sa*TGFjcIAioZei;kvFI4L$|Uw;!m=Ehm#Q zRU8cI5pbCPJUwC2ImF6L1MpEk4W0S&cFJ%ng=}EFe}$^rayWn7rW&7>k*J`kILBoo zhr-Y%de%9hZ`2{b^JCfJe)e|uFICg6M}AI~ZOlEjR_W$AbwfJ1Sy>)t4Pro+)MEz^ zEs^`Tsgi#dYqQ=GG=C(F^Qc!GoLNHXx+(8b%|(A z5aj7o&DyT`s;=lTXMc=vZRpLoDu~BG3}HEH?2Yr@N;@1D95I%hZV@Kxk{<(?8b?F1 z39mL7XQ*$^=9u@;ENguT%(|rDv16LZ?qa(%Op%~8=X{@M3Z8u7%&*YIM3{~_uPvqa z8{NkGej0;kjrW0ntPnocDa5U8iOYj}Kq(OVtg?cu7Y^ZJ?(1i~Gf!)wx91j8ER)tE;O9wNbIu zOiWC=gwVFPXf*ohJ*Y{T373qW@_u=*G?BkB^n0jXvr>;5PdjAz^)tc^&R=C5o)%T~ zV0M11NVc^x)wM-((FZ8R%H_(&j4Cd#Y?eUisGLQSG*BSp+ImO3Vv; z*MAH!4UtEG+t|~SU0o6*#FH@{>_D%-c4^}BwA8wP(CN?rUL(m_{q1if+u80*yi~4@ z_`d)O`2>qR{fDfP7u|l~wdl{yNCF;0^}l(DYD~@m=<{0Wdx|W{zd=I%fz^YwlocHE zzst)Y#?3%FTrSy+7|83+CXV44)WxTj&XEWR|Nmdfym(N0=ZdOa%h%n&JB*AJT&NtD zGaCR0aXJq^^uOzaj7CAupBydM?B|EF$b9i~?VNYO*w2b(PMXP*P($jUQc|*1u6&Yi zYn#XV$n{2>FLfdXSKtgZCdLWcB* z$TsRLveMz{Yy~%gt931r_)1 z^DPZk-#J(~jsyxeEY`u_+Q6*$|NJch22nfRP@Q>nsv3b!y!NJ@YILXcLrbw7t5t_w zmArnT6^vym^`Id6ApN-y*TI=rwk-T~rRh(8J=#$B+@qSW#N`5*pum9rL0nu=0mS_< zffPJpS9=PGKZ92z$(R;bmznO|$r}^^Z7xH@!`la5Uat|v*IX1=FYV$qfiT3-;NT`B zkdxcoyp-0y+URo|O-r@2yA>P|0Q!sc)Yab=Alp0qb{}zYJbuL02XR-sWMz&paE~te zAE4*8Pe@A2&9gmbV8f>B<|g$$O11~$gXAi`CbLF36PO`vZLDqYSfZcvom%>~PPojt+NBr==U7?vG9V z&$~nvyU_^E~tq%dOJEwr?ZTbY5v{Z!tx>-o+g(! zRc^{L+&YXRj>T?VzIqjCV2L;jgcmA+u3*5+5C23nQY_|UIEKjbGBxeY=*(jI`eu$)@hhE!;%vcH_c-xS$nU2%Gu>kB?`>)- z2CQX2f!bsHSArFw`c3%2EXVti{=vhft5EK_a(z9$V!KhM|L!Iki>HZAehDpr3A5bg zc}|c%Iyz_kM$GrKPmbmkdpK>m*_Cp*zp-LCRMWvj=4}bBXHvN{n2v7tl8Y8`7Zi|% z04aM-I||j&77(oTCB%Qa+%yMS0827L*%zV64uMWrPbg(R;eWP1*hSr77o}wv#g!Q` zru=Z&)yxJu30Zo`IuE&2Z7OrQ;8;GVBZuhDC$#c8gQ6Quw{I(sZ*RZM@o;tZKbh*` z83S{3W1OU!*HU{kb+lV~x-I{a=lpA@_V}Le(H)O5(D#0Pd>##zm5m;({rK^o|04s( z$L=4nPHy&uPD?Yv#b@yD5Kc=}yo82BctD(Dy$lG z+V~U{Kq4dC79*7t27#IwJ$9@xT4`tyqs0ANZ zR=Snu3-LtsxXp8il2pev`yxJ2pawK_I;tX|02dU`BXOTkZc;7Lc2^)0gmKq7FUiWc zqN<%6fU>i>pX(my<__5a+Tnj~LfXIjV7`XVeTut<4 ziy8=@I-G7nP;N{&;BojJB=-BwY2HVbN=TLXmk(8<*H>wuuCga2B*;@@Ik>8x9tsEy z>}`ns?ijBIJuU{-d%Xgu_T0>I%+T{935ic%M1&UDX?@k$Br;k%uu+QnQ0W=*KPazW zCePRdW`BL6?BlH=skSGvL&bYuQTv|()ZV*X`@TJn%fQ{dbPR$x6lGl%FINKEVYJ&t zRH0SC{{_7)=%}wRLArbunb_m^=WP)Xe$>{|zUCza6nX6I|LF4DFu_W7l};~7Qn0bk z(aTdb_Cpo`Z%YNWB1BrYq;DxaVs)rQ4 zjZ!__v!1VxS5nnt6Bl*(lG2Q%_|^*~ToA+)DXCFS8$-I5xX)u`rL2Tmjfk7d~P_<-s_Vyw^l>6cSg^L$~cw_sm zrK8IJRic3%X5sp5cYiSG#W(>^h)R)Ycc zOZqylc%0?Ilu=o!!x#NH-?(q(vIg2(bZ+})TOE0qVs7ek!85ZQHg5hnhMi?TZ+B6J*)M^a~I zQhxn48o7U*MlX2Uc&$sp@vIgp!(gf;z)4UxHqcW3y}@ zT=c=~lD7j?e4PpYxzzlB;((`mbBoK@>DSYGg@D=kCKMyN>F)IpA&`f8tdj0fO-j$e;fel$fnAT#-ZP^q2|! zs7-Cn+}7xsXF2udX5?SFe*%E$->bxF>f}z9(rdPWR4vPuEzmhF^n(0UHid}c)+YDNJ*ZSAk88uk1-*;+gBrTK*>T;NMquTKj#a<43jtC8K0FEI()c4PL;Bm{%C08+6?+a;T<5U4!-RqDBcrR=r(Igm=jKxO zzw;2hU(Vj+{*(7yETR<>m@~GA(evW6=@2j`0|X&H5Qn+)l% z(uWBgMl_@#b&pXE)R1-NQ1Uy89?k30wVvS<%6Z!5CX4<)(`yfc`@YYUfB-q|5%CA> z-j|xiE!(qFQyxnb=ugf}dhN}WwW3`bEO5^}(j^a$+WT|s8C9MQ72UO|2c14~+e}mA zQQ_J$`ondI&&j75vvdp=0s1FD*v8@u2iR!`hI1aqOE^9O$U;xgC9Y7Wwb zlKDy708eafX=W04nJHFH*TdmL{dUc>9*$6;Y%XnRHpy9H3F6n9#?^{unDb$P>m1Hjofigt?X>a+eDti=8G za?itD;?EyiDWGCtI`>;aszJ)f$EOv^cnA&vWPFVVt$ul(-C2h8q?)RNVTx;UYcRxt z4}5DcTQeI8InYd2)BFw<$Mu|lY$?GMF@*<(>t2ICGXdvG5!DRK1yithK|5j}?`l~C z)jRy$TnbsZq2U!K1b1cU!Qk^(d1R5M{t8zsH|oCGNq41}WU}-82JCNe{AlXk^7pIR zVPIGOa;)`hnJ#;M?)c-g`Sxx4U|rgqm>jhyux4+;yP*c|)0gsN`qVP=3!sN2F3w*l zDm)wZhc{q9@}X4M{w8bpCfuPQ@jG?;f$8v*&ePulbKH>$;A-S#XBQ3M zV|!9T^g0n(>CeEbLQXW*)vX7OHo~D40Jp!$S9#FN&S~dy(i{m&%*nO}1`@R*fP7Du z_LJD%PbFGSy?F2?=*tmq2W8Id+fw3ooq<-qz6HrZ9Lp&<*+ETv2wpF2U>^(897b73YWEyh-#3rTKXC6ucgmx(EFrFGbEE z?R^s-Lk+u`NM1 zy*5dAnUUg(Igip<(g13+)^pE0LnZ!2^`r3ij6}DVj*gCsFvZ1jY>o6Dalg=b*ucFD zfU!iv4rNRt@s~H)B;}qM5&ME`O5J~L=`*3h@l#z*(#id3xCeqCh@}^yZZeL?HZ%*k z63n=W;jU$DLifHe?G!1Z5-RQ<`yZXd)$jH3g1O)4wB51=HM?ZSbX3&r)RnF{j$97%`4-hnm#&tqi@Y6C8w0Qs)Q^%Wy`j<`2=xLE z>H*6`Ss9sYrVDllpD$CHZ=zWf*T&0La|V}|7UA1l@&Z)1Z^wm(Haey@07(2f=-%-_ z0HzyMwREC&R3E!|j25Nom7TP9Stl=b$9G690OSYMFU*Xj8W?P9zJWdWupLF#w2KQ^DnRJ>ti8x7-O3x>}3>;F821A>8sWim4@FZrIVx~Dhph4_4Wqr zwz8P(o(D3B;O|e4ZI<|aay249yx9-pu@#2YpH8cwrCG9>ECGUQOyyR(I(kqL3M6^@ zD_@!YbbcJ=_~r77M2T^0{_Gt^O7~+r5q^g__)J=M%I;?V)%!Xo3d)8Npo-r%$z1uu zj`H{^O-GTr{LbZ9jX;28=H&u*UP+Hql@bDjz)A4F1cG^g`jzTQ+vWI@SKE6qkb;_Y z411`&6e>LBurmmS;V~Qqpimyy(K&2(2Z8IwGYr`a)A_ot=9Oo>C>f`fzlN#P#~ zAL~j*Ke0VWjv>iub$31d;Jo;D`9pNR0X|ZVjg9S9iP`n(yo!qb=!`@f>pUe|&rvi$ z%}Zh$O$6w>Ac$l9CNmN^{G*sGOACvd+FAgjw86-OVgxOP5I6qH1$tfhm`s>Ha-?6M zCsxPBx;?wvaRA@j_3bf0KM(A${RwBM$r|e-7=VB5aGF+#bWXo_j(+t5Qz&DFsa?=bE$@*+%r5fXiQT zbA7x3reTn*cCLD`J{;~f-)yLP`ZJDkIO8#MIPfN96i`^`#XK*NL2MQCSq z7prlKvG3J8p<%H*!-!O1H46&%b5H5)_ok$jMT}e{>P8MoZmMrilni`=!N3WfuA>6O(tB z!+=N}g!pswgV9MdEvyH-emk)8N%2x=y8CuJ2lwBQsQIU7cxfd@4N;KKsFG@85lj0# zz3=sBV0mPr=ZSRh;-CDWm;lka0AJCr<$~O{7u=fc67CcMKs!bB!tSUN(eMvD^9Iee zo4T4&wA577h@MPCPrr*Ke&F5BoIgFQG0)L-?XkGRNSWLWcMFSnvElM2F0vrQia1LZ z%mke*?Dkppw57m`6FdBfy>~?=pQM=zH1UiN$DMj?=Y&qDQFF-xI_(eu+veYxMcE1d ztcgEhz?#Pr$B(e}#Er2^?;NJh?3`>+qKdF+nxKF{h(=9*0Q2k%Z9(@F$P{UA>bqdF zGMJf}%|((ke}0{E92^|(IRf>?lLOPlUatr(3k%ET?s)e@#vRa7W*ICe9Wyf>&cTFO z+j`WWUc1vRRx%)U)#M_<-nuQNS#<7gUmGx#qHlgbr>{96tp273u$`DQc4;oRv(q#I z?got4Q7z!NOE@31AECj4J7(0ROtiL|F8oo)aq#eL0(f+x$`N!(jMsPsR;IWfv{-sQ zeE8ssrMPK-KftR7{4nEtJq_Lh{BQq!1${V^)y{h1Z%E+n-<}o2wsn||%H<5ki#gtVcu!_=@!*dh zKitKczdR~{!8NPq*)!&sRlD6CZ7bL47^WYYEDt%#`n(djl~j54mbfU}lL;wO0(Tc- z5J7dO{Eh#X{448sZF|`Fx$Fhi+ucUTau4?R-N|pq*O)BY8ky>vp~tu*E}E(?38zFr z;3@@l!dH`|%+Z1Zcsx>Pk2w_P=r_BqPOIGI{ zY<=gBYz5%j(tkKSRMybZs&tyvg~00lsaUT!Io1NWC9fYH@mObV|KllA{Y{=J^=#hG z+E{5I3dsQ3C>@@@2DhF3?2x;0RJr5oN|=*(c+qJ>#Y#aZ8dNX*`)A+f$vW-4cnQm) zFGfu*!3-S|>e54;&Ql`gSBelEAM79s7_EA^l_!m@-yI435x9v z)7HSO4sdrK?s#Wti+k=AEUVD&lenO&9C>Uxz~B!K@kZtp;d}$T*VrLE@dPw(_Ll0| zoIBH%URG9CRn^x3Rdysf`$70bexxoB4R!`DW29xVB`{O(_)@B`VZ{B&RX;*`f$ahR z*{PH44jcMRwLijK5`?4m`g;6Kip_K%YzY1<^N00u3V3jx8b}ALv5If-EX`~X$0$ef z7%c^LocBJ*9s5BCV^xkgSfuqRkBPC6e5Z5#_}XmDece3S{tAHBpcKO+Y|_b|pc`bT7LRF3HgCz=Zw? z5NwH_gsjc{?+e`ymW?mo$+fs+PgJA8IPz#-V|wy8Z|0Y}Vuis2h}XEK6JS;O7NN0; z!?p61^y2sdD%0+j0s(_pYMU!fls2Pf%3P6qOiwn~DvNOibwNQv3kwThzFeA(5R0-- zkchP|MF9xkb*?Srp>o{O5Q9zo%_7_RRzQRTOs`I9Ih`i$^n%XxscRWB(4bYl_50s` zkK1YCm~%T|^N}D8!YnN2)qiU5#^-a&qtLuO31#w5@4i!`xr4{mGDb@!OEnZ-zzy+B z4aSA0x!N=^JlxwmIYxghgod$!XPoWU639-kq(jFfS8b3YyW)OYh0>~bD zDH#|TOg5Y+T^Wvqc~rie)P*rkpFK}{g^Zo+5l6pj4(_GvqnlSwFotz-xW1a9u42sl z={nDnrQQMr5im^Mm2bGZ9ur!7Yl3H+*hcVKK?Q!-214&5>pkU-7&@gcexZE7Z!=4~Rya{qql ztt*B zWr%#MZ~J5aQ?u?y)76e%OTn>t29@#gF|S_#WZ&)1Ebq}}4Kg2R*F#K3C8@9LPIM9v zjB9gyi;d&H^^g35oGd=585ozcaD`IWtgy<77vv7iY59WZ`E3=F<(*A??oQ#Db2ERv zd8rvG$(ziGlU8*nilt`J)(?0*{etT0z)1_xms%e;HZ@l7Q|o+sb{w#cgWQ&=6|eEc z3o_)B-22ND2Q+qpoQ&MsNSX30rOZq=2*6|k;ts%o=f;d;*@>R$*;%PTQjd?xmbCMw zQOG-DiW5e&bF!2W+@^%{wAj1OC_vyT#|n?GDo`$aQ%ZuU=*Qsq znJX+IKQX<(z@{GTe7L1X#*4J|gPmnZ(QhQLX}EQtld;=_#>kEO*`@n`RlsHf3>Z}H z!$UikcGOuRw}H~L=Q;Vb#`tXmM180To-ZpbkfvlQJQ{txNGlf`W)7Q4mmaM#&jP0m&I8OU^lG6#>a2IcJ)jb801LBxjl+nQn4upqpFxJLkLi zjd#B>-h1P|{zJ#;-gcL@Ygetc=3H~uLBxr&pxzGdHa570jGzKI@H6#^hT4=Dk^w7B zTEK~=thnBDCAJGkC{5JnfYW@MW{mX1X?!-`gQG;>RNvT`*(A)O&(ft9m;+&C7wl*y zZ8beXWM690CzA_6s5YBD*z6)IlI=iRru=8YQK+W2_f}%M=;^@ z*Uy9R1uKZ0RM*uVWV?bUBdhlb=-c?$sJejuV$GH$+jPufku#yGaJ25()e$>G?1jS1 z-9;0fLEn+AfVH(XE+pWldypcsE@6JJ{7A3%QEpj+_P=|gIL$^I-S*^*R6tEf(4}v! z|KrVF{6UM@4aPB3*!H^z3HY`l(qxHd9&5`wR*~gl5mmQojKSvi&N^tUd=QxRR2w<> zS83bJvj_vbEBAJr$Vj}qqt~|0r|IolHOcWugMC6p${r`1!tZhqfg0*lQi`Z#_~9Kr zkSN8Dja#9$3$9}<-{**;km7zJ4&pu+PVjTRBa?dX+s&RJetg}P*p^a7+u;I<{GIoN z&VkN3-2Hekk=@((WXa*8fhBg0TU$N#*@pt)@Ob0S>C&Fcbq>Ve3Ykhoi}9Zj^ZJ~+ zaGkGL)z#x^-lz1@l(mK!WYYm{{;$B%cCgLwk$n0~P7aR4Xax~g7M6kje$b(lb?3h4 z)Ycees#sHw6gb_Wes|WeDnR?9`+-=~)TQSh3eOz=1@qp$dvk6G&|LxAmeJs->UgdNK^|8r zq3ZZ(NqKb^H3CruVp*c#QD=E9f+=`g01msz_0=V%FP~3W7}$AAwiC3ZBd&!3tN2+}|~eB8^}kHI6&zMiPA&fh*qvy3u0 zb8XMnO@fHxuG~Wp z4Bag%Q&VVdEK@eQ(%H>S-SE2{Uvc{3-%ZbaUv$Z}*yd_O5$fZ$TC9zUOA$XX=#w+_ zbolbJh`bGt65|HjbPQmjlYkl~>(Eu=Nh{F0u|Gei9Vbz(wbc@|9c{Uw!2KAb{9AM~ryC-LMBmfnIDfu0*>IFnn=A$Ed5d&V0see{(vJSX}P4qQDhjw6%e{ z0T!jN{Zw<4c|$-^z4dJU0e8Dv=dNa}_smUQ>UvYR#gh2X!DXMCr+3tT&4XHLpe-~i zm+xLTi?bkQF?F{htya*vu8NWKx%iwNsYgERfZiWE`jpxF)1Z}jjk}Yi(3zCng5t@B z@)ig^UI~+1ShdlH{so>e?YjN_3-O@#?AzW~Y95IuHENTg{kHF&zxeVm3^l0SIUBy3 zU2IWXtATo-!}RrS@j`)0&o`3aYLlwt_z(YR6vaJ!N^r4cRQ7ABg*Tm%y)G>jI5HlV zxCx8-lccvsnvfwcyR#KqmiH*nr^@t!e!xeJkIaDb4~QVaUgZR11f}OzBmcqj3S?sX z*A64d1;r&LO^mm%Hx_puFcXF3Z|3h7TaVlkn|%Zd6FE6KpKhOqiwe~gJSW@l&bUicGLWlk9KJxs6J3BvL1Iw1yZLRQHAYEvXu(=uG+`~;xE&PHZVNs?I zkQV|_Tc;Md9%ofxc0u86)QAQGwMsplTI_CkPG0VEx8d5(RXo z_vY#%Z=ObsdND{-^&{OW%egB^CFt6{!H}eq8S`Zm+&!*eQv)r)YD=WP4jow* zVBcG{9aS1(O}<9iwKIYCI@zX54cKcP-`N4Hyg?c>`Sf$-Wn_cwTjPQ1{<7eR;Q0QN zAbnl6M9-D|!(6{j=Pr|##iQO9s{(bVUaay#+q!pViU`4DbmJcYFz>okkZb}kJD2EOA9 zZN$+T9$;Ak3CiUS%5UGM8-SB1Hr{Z6pAWk?w!Z!~b7_NI(b$NgtVw`x#OO%7S65j2 zWUdz_@a>SP6R95T6A1d(5$hGF(v*csTBWN!w=SbGM`EMiO>Iv1M`ZbF4p0fBmQW}z zs?xjWfA(VfAiMEoSF(taK9W+$xbSm|fUo=UIpfd3d(2p8@C^N4T`vHNw=T?mvC7x+ zSRe-q6Ktq)puW?=8ut+YiBtDN}tUIv) zM0Um@ChqG00qkT#Yb75bp8qq4zl636YkZlIv>c?g@x4VwTma7a56Gs~n!#bdX}%)x zU$&ZsffoX#>4I(-^SlspQR#x?kdTni-{?w|L{oHbM%mBoh z?Gpe&0Q?p}Rq0vv6=$jq#BJol)j0V0oO|{i9#FWfOn-Xb-Yp^7Eqd5dWKlrRF$78F z_sJ@*PT(UVChvYBt80VH8h-Z(*`Qe+9 z5-O|eiMZ5ER9A{cOFn?p{d-?hqz|pA2~pJmx3;P)THxT2DBbTFE_w!bmk2PFrsvc) z>aM2^FyE_P_*B~YN%!#S?nPNG-c{r1ypbn;l}UtS|yDO59av)g$( ztC>6obtLm3@UuSVMR#I%Y+K!cmY_l!PyFz&`0!dFY-^I+h4Xb{YFh)k6*C>M_lB8r z0}#GT(}mAWwbYTKUT1&7R44Wwu1t9rnyOK`bO-(Rp#BaXdCE=A*DF@BQDu*Xek(>G`iyoLFL) z&-SaT|E&c?>0XHA5e5)CV3pLHY? z%ZmuRSLv&fQbEdSbV%oj6b1&CIN>@58;X=EyvpyaHeY@L3zvM1t$KB3V51d_nAPy_NDusAKu6p-nTp(cXA>mTv-qnn)@3w z^g@nIy^~_vXizRWtgF4kOB{2oRQK}gYCf7iLi*1-TQX!En%b$xgg!G1Po6b@kpRhI zr=KC#A@KrlH^L*QuuBKUafy50|j4VSaZY zwm*J%rlG+xnY+kWL(qas(BBRzifgd*8Y%RKDH2&^O{A6aFtCRg%2(W`0(@AJi?Eel41CHwh>1t+IUnmLB5%GnV6!)7^e)a}qv7f`at z*i9Mf3Aw2}m!xif8|Zr{2rW%Zkup_N6DHyM+7h8yV9d42IXbqzC{b3{($CFX_Y&1< zNk6!zA&(!B9c1)Q<F~75U|^g)`=5I2`9aFJFq+m@ zR%pNi{((5iytzNFA79=1NY2bQ305sexs!xZ0XQ!{P7W?C=vp2m9D>06Kd=A(Td_X< z%MihMCH()-!2jEy$oyy0cfaYdIh`mrcrZ-JV>c+5D)!I_8gRuW-p^A`t#|=bj49(y z?4!YWWhF*v3i)S@A5)bWOhkO;OZ5slqDRa*NM!>#fV=8uk1_^{e4YKm2Mp3MFdl@^4gB@*&AePf>zN+7whe4`@xAc$_@<-gYnF6|cHzw0_VbM%z)1Y7}|oaqdIGW*w$ z%wJ#l@Bgz*K43L_Pk8!RB=$zdw-8IxM~sXGy=KB#QZ#RSiuuoNGv6YTPVADic=4F# z=7Jer#t02C_`j20NQ(Wl^0Wka@nQ>zoy9dDpI|yP6JY#!4t|8-pC9=(+Tc-mKW}oB zj5cC?A{9*0)4$~ss10I)M=N;!8ZF?)PAyfnWT6kQ)Wu5wcDjh^zvV3EYUr(AgWK$* zOD%w9uA_S3u*E3Vpxo|pQU65}>49vFeIob73@4IOYAlwk^0IgUaL2R0fl0^=mP^oK z#KS(l58qtUlPB{_|=X7CUoHRAjDh2ZNx-3AMkdRL2Id*>>>tL^i@#VOpWldvs^+7l< z_z&1y^csh!fAbkoTd-{v)G82~f>s?*p`a|YY96km;VorNHGXR|ErJB1DQ9&R;|!Yc z3X<-V?WtBvIbXd4cKyA@dOCT_7$8K6ks)r;mwr67;7@TiY_4Dz-7Q0#_ON?PJqHHE90-e4l}MPZJ8ChI)1FdD>u=Xun?^8UCt;f$;_Q%#A^T^ zbFH#vY6%_DlM6Q8rdo?JI&SWho@Hv-^*-vj)eoDsUBDH>vGIt^Dd6@mTnJA=VR}!a64{_{C zOj;H*lo`26CyGhBS`It*dhbo5zf3KR&3w=Mst%I(Am@GjH9b^^ToKIO)grt#@q*u# zP$FX#!;)UEp!DsnrH2&Wo1pj2JD-CSkBQfDyyt8&^?EH{Uz$Xt4j~LU4_fcvyJv#u z!kcZ!Es+5TMQrSpwSwl^qS~M1J7tu7_8AO4#2ZO!qM}ehhN^bj82XMR!4`0E9p4=B z$HetL6jivYGVnL%juL7)9_MR2hMf6bjP~$b8$P*qtKD zr7ySq1W)&*3CI;3bG^4qP^HbK@#j4P-qw(=J4}<``iYrh8*_3JmRT^CzU9r&D&#Oy zNh9qMjrIQPlqDgqB~vDFCf-dla!V-{o??NEL5Ox>SV*&1mpy(_*wd-Cx#40|XL)Tt zzz2M+;wgFunZEZ0H8%_Rg@h)nK!C^qyjfX+*OXKi%eD6wv4TQ6nQhZvb0LR{gqEeF zV_QWw*3l9B=ug$)&a68P4} zrlvzlNOM_PS>X3a!gBti+UHU;QrOfagX;8Z6!f{KdDk#N^729k3ML2BybyPy@&I1A z1J;PFNa{SO4Y{pD$sRv;-&k-HJ(xC%O-7z4_^s}*&o=<=AGNZygak#Wbd@|k4Nk>G z4wpi3oMXI)9~L!+_|iuA9&cUY#&LS zlOkukoQ(uW+l;^tPeJJ5+zyIf-`WE8)t=7K!QjsN{%#NCc}F@Qh1dR4`SUlIr%f!J z)T>i4-@$%vyoXPCeZ|W3gsV6>fx@0EsBT;G;?Igy!%R?Z)KnefUT%9824b9RjXQ;R zn6kUWPL2-?8`ggO^zVQIT#9a#NQuv7dmI!~^-<94(7TZjEhM$Hl2j@2KYit*1$4}C z{xY(#JddOGiFpZQ9`bg9khW0v=5Hz1wjYg*jQp;ayA!KSB_t&^7DoH~ujE`%jV`@r z+t)bk6y6RyGcz->+~5Ik55faDCI&{4%iS}=YdTtE2!#0SZU+nox3=_kG?X`?RGIU; z9jQXk(E$~{s3m*BRlJf_tEX(#;H4h6Ew9iUZw=x-tLd-z=vN~Vuqtj|`u>RtIlEt= zywOcSx@^U`JMG9+o(lPTedHJ#Ex#VKJ%k#>yU%l0{o9meEe4ORVQ1MO?q#Sgf)tN^ zqj}e`Kxtk$l{KIWNQZXrF65>k?o{2eBO*#@qDxIlZE}WBH0^!R7rhpQ40~>bZ%E6q zFOrUsWT_T->GJ>Tebh_x<^QuL>?$7Qu35L64i_By%7#Cu{`kZADYY9p9_ z65TTCsLrJe$YLIyu^aGh0F5LHp5Yc)fZB(SuCDk;0A`=H2MwXrZTnnQa!9G)0U}wl z(?*COVU?esi3w1D&izpBSy>F|kLqll{yf$$qq3%W4$^cD`xq1?r3FAHT9I7*oBfx%E{|?koj1UIezr zwXN55(`biok8*Mf!xabQ`)9JUZr#IJviI&!E(g(s%^5BQN2MW6$X$66FSGWj>5z1| zr)Tn%NbfMOTTfWhFMgrE>*?k3{LQ%==s9LL7N zIXrUeWL&l0?mA(Wx>UkH99!h?x+sW!8co2 zVD=SSfzdHg6&A(E+m+Pis}_FtpObv+da|E?-A$*rD8dvgAj;nTJ@2mB_vj9&$Rheg zZ)$8za-P}2DQkCSd-8d4G-wyAu>myEX^9jTGauCLECH8zW+JsZ{HYT7VjqhJD0>2^ zLPw^>{c$^WlX;5nPV4Oz^$zlfcppj#%bAG!cwYZByiEz)4uWMsUMUu&2n4!4uq303 zXJ)G&3_x)Hat4BLaeW$m*RBUEbGFV{N8SI{0-A)Yeay+p^j{wIef##y4d&%G(MP;e z4250;$)%_GPmo)Xbnk;slB^y5-Hn1n@paZTKeR_wUV?0%K9F*Yjn~&|wNA`$hM&$+ zx-Y#hu(&VL4hYgr*i&*1pzNz_-}GtZ_Ha%La;rPN(3S=9Bk*Krs!UtMRx40S}%EvBTG< zpl-pv)+UG5b>brRma9HLKtHlV9Dx&r12)GvrB>2x+E-&@VgjfxA%BjC?(X5Ma1NHj z@OLW5xtKC0?y5gpBGvEvoL-$t*3))gSlMo^BDRj#K8MGF2RwY7A)DS6&O0p>=ruK| zf_0h}XFWJJ$IeMzW7BjRkNbGBYWSI5`h_3^NH4nqpK8a7izzDaEBax}qN3i`#r{}! z>cV(dv7@qpU^0$C$zuB)^1|ZwB%2_STfLzO0Un-N!E(w%3c}AYM!Dc2?-?92S=u&Q zSY0o^MM$V%Y-SR&^c*;ibX2rwtv8Y(h=li;cf1e!uocfo9;z`-mIbhyS&k~ zpKJmD9vIXv$b`&}GCfftsY!AJioAlm32$%M=^Y_U_^WXGG*S>VP1oi4gtu$ysf|6Q+HDESWO1>c`XBHQwyGXEU4NMs*>q zqAXu_v18`IGJ*KfJeA(MReRadZd;%rFMqyv$?jOm?E8-GFxG&&JMythuCiLxn0bx3 z-_;>Js+}0>U3bX%ZD6(FO<9Wfo4KV^sx*8vTu)Tmg0}kAiIe6`1FV0_C;e02T(ief z9n70rEMlqlr^0X=t33Y`)nh{&o&1tQzP45PJn1Q0@+hriP9$%~i z&&;3eQOXuM|L`fd*{C}rQuM-pI88V<-qR!bnk7ccMBnd7?xWu;2c?RypJ#QaYyor! zzQWA=D!8yo;0YxNb0!m(m{jex#7!|nQVsdZbz?mm=3P56 zCUf*7I(shQ!WsVL>HOlQtYO3ftSk#Ia4wrLzuGM>SXssND2w+=3HgNXgo(evJkNsA z38KJgD;?VO+7fe!)Jz}xBcH8Bd6=g59<~MSrR7{6cj6OP=O^!e{B;~ku?#0v?8aPS zADoR9mH8O1#%0heHA6Y99iOh@cXYQql68&oe5sg%^rtXGTGMC+RdNYhet<~#aA|KmHNOvhv4xzVi7BC+?*(|WCXXNewa56- zrOs_r5k)dH1#>KeK+Fu=4ptqo!X>ivx5gVb0PSrVM)xjfp?Cq zD{#fKZ9WkpAvpsj)%Pp8u5bYzrv;nX_=qQNHGIK3G(4tfap6|VLp1z?O^fzB$NOK0 zl;5P?qSW`R`1IB0(8g!n{-e_THhIY~$Ic0srNZ z!bVr<`##ce6;)OG0t;+x9Q86yIAUGIcHX_fTdxdAYpPc}L9fx;3mVNA+g22QD^f}i z`G-&88>TEzg-wsG%7zO(E3GH*&aSL{QMpHzM6d=dpJh|{UKSg@U&~!wT(=Nj&{>FM z*XA+*a9nQi#ltzX;`0oAunqDG?^|RJoY@PsgrNM z$jixb$M<^rb%x(?AeOu>>+LgDW#yODdK;xa9Z;&~sJ;zsLO~@4qS!OdgclCm8JR|< zO$P`Cl7Rxxqg-0r>r3m_UgIH0Ok6U-<-Q$(FD{_zZ*Xvsz(YBi$A(azTpzk#_Qfw~ zeI6WgU-L^-n@%G{9me}0uQc8N1V52T|gikO|J`IgS#N7`&SXI8$kK7V7|D%KEKlXiL^=Y2Nsrl=DDT1 zO(A|qIES2^!=2|4c{MIE-6j`7@AGftTu@Y_F%ZnBvN#8P^uYU1MnOvrAV^NMv^DeP zYoIUnmClzh_e6ntNFj;1xGRBIkydm}!uimgofK{z*v+M)=|aTn5DV!(rwu!H^#_Oj z-TFQ!H2o1soenIZFw3Gv1O8&g=o}k*umxdT#17(lXz3n^)SM|MxxN9vUI}x9r+YKZ{7yMH7J1qY}YByNB%Gw%KWg$+^;n(eO*(>lfDiK z3keU39xOK4yfLZ%AndOP-AY%Bt2}?_@O#O3BzbRZjn(j86yew5r*S6Yzr&fVP|d46GLmZ}~1VK4+%Na1MT^Z>-_y!2M6+1%JzFE?9+*G$H@U**4k+2iMb z6^u?vnFeT7{?G!joTKI0$Gdxcg!k(aNoOp~5CAnPDJd#LG-`B&>ttFdlE+I?R4s~v z7Sb*oiyFy1#g#dXZJ7zeREkjp(3vyBPa*EqK4&MA6A?SVe)>=;-m`RD>?3p$y+d`5 zU&qXL;nQ0e+g{rF>FOLVc3|IQau~R1_JdAZTKW{ez$~Bcd$?7|T?979i<3QlZbgf4 z^EyYCy5K0zRo3KhiZ@wGHAW+~BgTz^^w^_BNmxJI&aaC0n4P?kBz?aEtNE^P&))#` zW4S05^g;I?5W&-`kvZ$`P}M!x?di4IP8D+AUjkwO)v_dI=ts-VUj|g?Q3Fodxn-AL zx;3nL6dZUegnsED>Mk6@ql+z*fr7xP00cbsE7&#Bv%S3*z2EhvT?P+xi9oOs!9>eQ zUs>f*>;J&OSF%5TAO29OV&&b1=8+cS-T|@b7Xim-v@b*0Lm!S5MzI{`J-V z`p|v-SL^Td`lCmr_qR5W&O=wZNn`I?FM9iWD81(Nb(z?a4#63Ac@|@iBkmS-uDFnK z&+9J?))ezLmVgKNzL%~vA!6Nr$>jsbA+MG0b#ug!`?6Q5bjMGyiNohM8V_9h$mdu)>jqOQupXr6rPsI z;#(3doYl#DWxRP0;WQ(NzuI4`XU+o--#c*@c-~c;)Y-51wrqS|){RZZeagNo(5nh- ze@i#hi|OpAXJE*&%=qp}SFmjgmQY{3dqE1z9&AuP^c8Kr?&X3Yuhe=JgW%f?fLPnf zWKdzRzyNw0F5$#$YyZ`Q-p2n8y=;n>3w<1|@#nd~GyxYjYgrjV6-`if?d|U;UXIg( zJ0rBP#T7pYMrG7Dc?s}aWMM@T#D*Nj$CO<%nv>1zdAAXID6EaLJaUl4OA*`;oHxrD zL0@iP-@&|NyU^TV8TTzc-R-z%#20lD8y6=X?($;tsTpW!1^rR|S0BXUhlrzBJ1N3q ztqpqZpdvgw6%XTmjFtfMg`X@0Hi8pSuEc=Y(+z+7Yp zG_oZ`rM*NgdNDC8q?f+mddpUppbcD0iXgHLY^c(!0YHfd6i=cOmG~?QJuZqtg<)tAQV6%%-|V6A5o3v0$5qu?;8tjL3@A#L@Lm)F zNXFVD){sz_op{P9YbMSkYejo62kjN+T4OY$@ke!S*`!+-4@}Q$J1oQn%rqdM+NoC_ z++=`Xr1)8@F4QtZUJ+2O8vMIAohgqm5f?=B3rS6{*zell{ps2&xwIE`pRIg)7 zzIk&F3em-$NK%P04`z6R>;%ao624JZSvBf)LXK5E1XARO7*cOvt3G}@ua`n|q&0RY znYVPN(L%075L9O4->E1oe{VhiZ4;Gob&MhzL!^zoY(5IFlfPb0R$#^HF*Ae6xb*c8 zmTFSpB4($A2_Q-$Bl$~u`-{+?HL~pMs~S-f6hh@*SFuy#60T|tYAY13Z0X$Grm!JD zH|2^M5DMgwHluD81`u@=U0io0x!tOW|D>guqXF^krzshp{N;QxJfrL)yW>YV;b zqJQ8W^mNwB>vYqD>G^UxE*VQKVQGD(0_9PT$Bk5jRMj)78Ua?zaAF!b3ip!l`3 zwT2v5sp=H(HT%AkmeAiT1qT-HH4@EJm6es2w#W@izX5mSu2{msBP$egBb`KRvlYls zQ+l6gc4<0+!=4wGuye7e{r*?5-8at`W1P!VgwJ}SS$lgM5Eg6&YWK1nkP&%K)1m75 z>%nAZih=t2?D&%=S&qoynd|m5Sm@^=-c=QVFPFQj0c|yxc6l&~!e93l4uhyJLrAD|%TBbgp8wK6E<&U~`ix8Q2ALD^! z8E3Jc>-jd>MfJTy_w9+`*c1AQoX*|Ca|?9TZN@m$Qmf%5jdg_z)pb)6R8`?$x(q*` zFUE;oMzpS|o@6!eQr(KRWb?^z-0pZO^X}U>`z)EdMOy>~wS;bk;|%)*j|$VUHnrAs zy8&U6=RYrMWI7Av6z$C~zb?k(T27Us&Y2+qaRA1*E$gRBx?UlEc;GOt#O`e#ibP+Q zi8k!Cb_=*54mJtdfeZ^V39+e}$)r!VNb6q{R(+iQ#bJ%#6T-q&w_xvc*ImYj^R$LS zUNNAjm-E?#+o%N~xiq0M4D^DWFDq7p7M~=pm(C@$2)?ycJEAMTJ_(5|yJmG0q7fmn zMVg%~`(JWajmlJawp`w?KMfSj4eVi{{9DCWvC@*-Zpg+c`||#{O)tAlX%ZJd1B3a` z&v=|2m#o}1HgmOcFIIv0ZjpLP3~7-nB$?l+3+POb7HO2pCD(OycT0+kgBIxKt7gqe z5v?f{1} z;_AA_ol~Me7TQ$`6NYv^578uVbmH8{uDCCZtkW`01i4x)@~7dc4aq0z_|g`k7Vo#~ z4j*6*CY8$a$!-MWJUPMqZ{Mo&R&0PnTBGc$q>>{;PP^cfvPv{a@1@~HPoFZKs%yd^ zd#~6juffyx{k#0^w#aQ4!DvU*?QuqS=phWhUGh9Wx{n0@{_G5PjoM?@Dp9xm477Uz zFt$nVwlLFVws(_X`~Y4PJ|LP4en&(zr+K(TfS(Uqx%YlUnHQ&iuWVE45-Erve@E5V zYumQcK700KZ0w4j4JmsF8jX-g3ZeiR2=1mxKAB$SU352kXP(a&T-WfmOwSu|%(3wF z%!ff2@dm)`I7!2p@;0%AR8fRlQUw8%pbKayk^-qLLOOrkA%xqzR0SO@3hnTxr(%L)w?jzTp;x zyIY7%{^%HOo%cQtGs>xG|0#{4l$hsmpV6^HoFgm!4ra9ZML-hMh;&Hw-#gVAuf(To zd{%!v&u(e)>5hP|Z|sTIspjRTAr_A+PsypTZr$P>8*C?OYij}tThXV>xY@eM`h(ib zX>gO z|K%^SJ8z1fXy}?bxkyG32LO8K{Cu%Msmr>vp^{SOSH03C=XOdcYi-!ZWA4+!9~rI} z%DOAPPW~Q|6v3<`k0f0X%R6>@X6$sPl566<5W)_5P~nUq6|_LjLRn(T1tb=Q;IEb5 zzqpS5Ji9(!AO&HK>ywN#=VGH${yy+Y(j3s=q;QR2Y<3Fch3+~Ip166Vdxdm^8-^Q= z-b0EY@`R{Dpl4OJ?1Q--n;w4+#GkPPvfczm{QPM{7(H;UCUa<8?`v6O)uK^qIfzDG zdO7c@+f=b?^Mpn#7g&?c^B|oaYqs`4^TT@pA1~Uss6uderZk%RM9Fwmi;#y$!<_|| zgP`e^c22Xwuz^o^bCmrN8xcLg*a1C1I80KgnY#`jZ|}-&(bo+vh>wr&i~cLf(C7$f zXs)58bQ`fwzWDdCF{TdUan_i%3 z^!w3hP9u7K$0aZte4coVXQc8LarQ|0@?P$ge9Ide25sKbiu;Cqo_wa$JqJTWrJNLA zTgO`}fZhr?&CeI-*QBbR7w7-tI;Vug%6pxOkLJ9 zI0)wJ_3>idwzTg#0?=cq>fSeRwhJOsfi3ngd0j8sbP?idp0~ewC9)47d(iyG{HAw| z5l`n~kn0b;Zjrx#e<|b)4o>yHo{$_vr1Cos#I|^zf%3HT*}ggKxiVces^tQ2CmJ-! zMu5`v3ZwgRaD3;RjrH|OHm+S+N3pfl4=w4w=PPNK>G#bEkJHUy?}R+&2isMInzMY* zBIM%yD<^y~i-DT@!2x}TlqDvn|LZSY%2wYsOA$72iyfO&{y-Xo#+JHcj~|EA0T}Ed zpgRTYSQ;3fBHk%*Mv^??tuQ1`4Lh!;Y}HIibWo2GM=!(SmmA;h;Bxd~{`l~o-wMGpHejE!A3iBnR9asyS++=o_dRtgRa- z%Ssef0ERs195FC32(CSlA>ildZ>n!xUpJmH{zS-br>`5Vt}T-OzQo!weS{BzK&%AE zS!WOQSu*RXZP={rP`=lh|H^xlA8S3zjv{`h2+dF=37CCv!=o!n8{Z$E8#10_cn{;l zwRu}HjQ~v*$LLU5^n?#_qMF)?&aY>zR()lSAUq0*HpaPgr$PC4@Nw?zzhvcH*prkY zmAl3F3oPFLZTMtuj7T6=J%xs>pbol7_s_&&REtX}glEMbOECTp`78!#srGld3_uVl zhx`x8Qc+GRQ?Ec-s>GjfH0u_i!P!3qpLe85~ zQ`IDA9l-EdVub52{-tA1|J5=SlUV4YY(>*e8^J~`ZuLvh+KmjOTAs_`!M{w<>7c8W z(fI*ov+z(^y6C5&I&^NXuK1D~-?W+j!g%^XF2U$SAN!x5{lHr}CgV&iaa}tE=$|9z zF;73Y$)?7s>ZoRbGx7hzPV`caDVwsuMZX>zvbD_q`62ClYl^Xk@fwk1fLOu!BT{7* zoP;eLjHeWG2`$wBFi%}2oab}shy|ONI;B>NRSVXzV|Cc-lBHB-^Ek2>bJ)3wt-@!s z3~i!b-@>SV@V~MwX-9(J09}=GN$_Tf_|_lV_05R?4^m?PTW0D1d)BS=J&fJB(9j$C zD2%4=AOY~#^j~0cxE9yHr@vv`4m}Cay&)oN@w&OW6?7W?GyUM5{|{W=|APbnU+Kb% z(qK5?<@$~-zQH4ZUMR+rK0u67gTeTF>I>5TFie$#q4x=P3Gv`Jq0{DSt&?ROgwvZmD(n2 zV8zmGr7jS`g%`$DssoUoU>C0k)Nt~Debt+m$ujO?yYOtfNMtKH60HED?47{>EuD5b z_w@At37vMO(3)_!xN3u6d7Cz(R8HHBgxAy@3nS>}!TX0XY>=>k`)@7aBl-TJ+RjO? z37w?gsru7YJ_&Y8%mR#$Z!+&WJ^p7xGICWIOonWw{eM2=7jOt;LrDK`$+P*gB^k_p zZTY-kS7X8R_6cmo|FC>N#tpTrn7pWoTV!y^K_I}_EZ&aw4Os|sg}vB3~h5PN_NVrl;V~4Amy5X zT;+CtiC` zyYL((4#7LeaC=sRAyA>pTYJ?WO}MTkGp_ z@ySQOqZKCSmw*Lg)6r1Vo4KG6E?PFk=4ZS|)-#owH^s82QTk+c*It*TfUcm##yGNJ zYK6L`6Pu-7_-WyF-6Fa0K~h?rW7o{B@T$>;c1=dJbvBGwoRxn@Sk4V-`5(J3m%G#3 z+Nsf3t92UsJ`)#Ksqz9X<|A3<<>k44DB2lS`?)1ZS`vJB#hRa)D%z5)XYCL;fF5NA zQG!+q0$>j*tnEavk|ztCsINml_Zlk~s4g^}z29FBa$9`KN=FCsPtf2RmDDim+p;CF#wX$8K zfZRDZr)J6H*dUgx93Ml(Df$?Lul#HSm71=t?7_plGw-cmYw2S>N>ob#R_HHxc~biD zE~!jaDcs{z?V}vi{Q;-pU@jA&GSnW3TjpUgmr;AP=-UK_FOmHcbUr_Oma)zQ7p(U4 zJmt)K)|Q61=m%>bdy@j6Dubl}Ys7k6OHi!3*;c2Ml~WJwt))vy8XEb-pOVpH3-Y9E zHSO+0=XRL;)syViG;*aaI1)utpqY-AmtP1I?{}y+3=%lGBPP(_fEpSjMl~wMwAs?$XQRGI6=Z5FN!@{x)1ep2;hxA%Z zT_&{gVK!M<4@iGO^$i>j(sY}cB;CQyESQ2l!xnV2Rp5(NY(V|We3LEULclL6Cf4?I zdK%bMOXI#YJx;)W-fF@|op7($qgId1Pt7IPb=Kat4VS=7C zJUo+*CC}dkYBABvB|Rn=-WOF6HYwOsL4Y|11!CZPbWuBJS0*I5aWeIYU5^D@I<~Je z{YKb=Aph$_6GZaS+YptLZG9G> zKJ4xhV^4d!nxNk*-o)A7S?>%ODDwL|Y# zDV*)W*ex;dKmHIbHHkvi(JDQD>1e!mN-Yub@H%Aat?_*TqIo(Y<_Bbt?)HK7_v;}r zGF5Zg7`crV@kH?SC?K6xvoa^YC&hlicU=>t)LMJJH*{%>kmDO06K)O_p-JK|AB5>4X)g@K%*8~?U z0i8;21v}_K*mx~`R-fmC`>=?Av(FVR#w+HGk$gm>UC(*%0+W^X=*4s)qc!d0%v%{J zqvk1ZV-A4@tM(A^Ui}=EEwSJ{Hb{lXz{rU7g2STe6i7w9J3ixyeCylTBNZ4I8lDJQ z>+gUD^~)uCruMvxpcF+9^6G2MHHc-_Um}{K73hNxCkK;ZX`=eOnZO)}jMui zO;P0sL!(}ToVd@<5V_55g+ie&@gA2Obw4N&fs2Vj791dYW$8yWJNJiyL)HMc;L{P^n{h#dSYES0&9u=t;rZ%|=i&(|wBiDDzI-h^V zf&c?hDC#u%Wr6d&vyx%tc^|=OYvzJDja_AN~e`I zvi$52upmQ~vV?h5o&`#L@Qc(!B!zUcHh+lwa9O5;+1iFME zgf1?U~PuI^-An-CaSyl z@FIo0HGq%EWjSW@;66cIDj@dtB(zRcT!Vl_?QCE3U!} z*R<~whfXpnsS{b{mRqkY6dYy`6|Us!F25#ad{7|6WLjYk7AwHMdta|=g~Y}oQ z1397FcshHG_X+!#PBrKnMl_uCSwpo5zD27YcPm`7u60DutPnjyOcmLEHO~}qJ{Epz zg|4#uXCna#RK|%wv}4}(AqBlchc>oS)cPw>y)U7v27>bBVA-=nudEX(=$_i;;|Vb8+r2hF%6oyK-pl#)pe-7Z>xIlBtJ|k4><2!tALC@61^Pc61OZe;q5dSgX_OOr4szQwY!i? zbBkDM|EMe>r=PAQvLvZZ{qw6kuXfhJ5qMG@6!bs1d&{^gzjj-AA%9U&Kn0{hQAq)% zySsbQUDDm5BGMqzor~^nkPzwalJ4$4a~aRG_c{AL`~98setPeZE*I;LIp=-NF|IL& z5A2K97qST2`3BG%FlA9|-g@6QnM4;$w%Toiu47c7^I&_S?eY9$5)c%5I)@>Z(B0SP zxxNg|u_)yxH5^D4R5Z1*p;GOf%AC<@=yN`o?(vwETin_*qv&?o+6Uwc8IfyVcMWmZ z((;N5U3yc~;hl(No)8je%>%qQk4eA!QYWWHN20YxGzf!yPy%#0Nkl+jAA*@l7_v>{)+ZT+Jo zY5cNXWW8efz;?&Do`aJESQJzF7#wZQbjh(ib)1${t~jE7oFyQP{?MFw61dpZYc{&{ zsWYJC`2MyEO!nRS2NkWPWU(3Bqi9be&&sb1OE|EA%PnP*(Y|MV`{-KXs==94;I|Qufzj<)+)hL$D zZH(lRr(qeurX$A9-7@W)(~*^`W89JAOz{e9YHa_{x=;NEfpAqq9>?ip#wE=;t>t;PwQH>|0>v z)HXq_RZ6xzr>#G;s|zOb!PJpDs4j8sule&eK(_O|k< zuQ6xzpMT0?SWPp?-LCe9J2Jy0Z+qJff(oBrOSirOS$ObR;nO)~=>STBEZ}B2sDB145Xgi5e$rW~&w-#3BxEa9>bT1q9f6-nj6Ts7}w9 zS_==d*&}26bxjd_D}cS-rnXL^Qc3oZC>i@_$DLVIo~vShgb&|55I*>21_NPW`xAl% zMl*f=o(o#;={&0JXoDH0{1C~%6D}e#u|O_cDPJWlDsFS#fC%h-@W6VbVIODE70Gq9 z*TT!Yk_36JrRm;KI6giOmU{m^Jz%v6T+2PrQtrV%Pt{r$E~B{}?UTKI`xfMf@S1zI zx~Vzxx92-!EF(n5wYIR>B}6(m_^WGM^^jNf~|wUd~rLwU)rgr@|=7b!R2c z(%Rm(d^(7EwPBzYYZp!u1ne&DOQ53%iD?X6$|@;USdM`d3N#eR5*E)szrmQMPr@IrA^+t`p zSvw16Vm7+dp*R3JQSN=Y{PbyfRkbOlslegU_UaTGp-6U3jHD{D8l!hU!UvT1qq$i{ zYXi9SnjXs$Yiq{)JsbTgUIEy&1x%%*g9e^J{e|ft5Wt+LrDcvzOmo&h)M7jCJ5RJj z#Kko$mlnQZHw1e8E?45fPI?1a_Q>fjbmP(!d7M3_Ld1%5T}^NVA#tSY)`IOt!3%j( znU^P;sDVznpYhvS6CNAPxVm-Q+yzQssBP5^e)9iRWfyDRC%y2T3iI)V? zQRU`G@1Oql4vA}(M@L`z#3Y_6+^ zo;92!kBqFwY{}(vlm##Uehv;q;6#{IcMMZrq%YYxXZMl=X!#CY{0-7a%B{{A;Wh^S zHXwy0Z+?4oX*sta|45E1SuUr99@TlGjuC9Z!R2<)jLL)pj2nJXtNj~VU%2xZdjIrC zcVA`)V+*V3PgY>N8D6G@`j+K&gLRjwq2etYZk4vmlwnEoZMNCPW!4u(3nVdoS zamAFX$`3Omw#VSN{N%Bk|fcG<8(IwLUWFb#MJU|IWQKX_uj!8V z@=i`QnK0uA2ghI}TEhp`=-;1e;pV|0*Z@mOpitTpd26xn3lsc>E$rmb|B3jy{J9Q0 z*@xrk1_q=Z;J-a-U*60-IZy)g)`uTs1?R7+2BDR;BGO&<%`NBPa2LO{2WFdF8{Z#p z|7XvMB1EnRllq3T?xi1Hv;Juq?Q7dD&j5`^{vY_OGCLGlciX>wDMmE16@Y`sdwwuK zU;YbUKirdvQRRh(d&^o{oyve>z=sE8MdZ)9@QAO@XVXQiid%-W$T>QKiS&na`_1B& z*-8BiJr4@t`{Q|$`o~d3l_@6gEk=o}6WhHq>q3>J*soB9tkUph6cvGVPwix%+DuaB zS{4e5RXg3_VV==x883z%C(!aCvv`wEHR6v(=w>>)x`b(DfT@&td|>StcyPFHPme3P z4m>T0;GPyv3=-{DTHdKHD5*p1_yfJfF_*DAh*<(X3R=c-KpbrU z@Ilh`(GZywK@`nhvY@bn^u%3zlk!PrDfnK7hOxTtZ9V;f7yI<-&;HnBTP-z!V5X8E zt&N=QXSP>Wzsu3Aadcb|CN}Hr8W5|fP5Epxy?+Oyn*9A*)N<&%d#@UFEKln~cnhj! zRvDh;Y^^@$v7M_Kn*_a@8LJ)3_Y1>Yn1~{Ea;{d$<#47uTvurnMO_8(zq|*+`yj~# zm+PTYBk5`ffY97WMvVb-SWpJwu{-4H%L9i9+J>`*=Dx;9nP4v+iw|UCoVQmGKxqVE z*4>(8e4ydt9V^f>nt5=cz!jbwPWXvV9{R{@i= zlj7VT#`)|EHgQ!=&!qP){R$Xv1U?(^r2AxcU8AFKkB}+z9cq6-MeyE0lI)K=ai_Nd@R8d5ar+iCq%%@j+k`^{rhGA+lQx2L&QmbO9vhAph_o~%XC zZlq=cbXqC}|7d4oxFMR2$NK`V6BoZy4Q6wB-#ZjNeeJ zxrqEjnM#1OMZu_Uqubl|u5AksUZiF@?#{|4u&u7H7OEF>P*R$V`#vtAn;6ZHLnr1k zo8S?+58^>d!`^k!a!2`vH^%4YZnox<`Zez)uq&D<7S6hJOmntQ0D1-AqK<-@-Ss$j zMA1^vtQI+4JY~ z+Gbndyzaf)ghN9ANyKj=YTBwZ>u|Z9x;N{3=y^NCfoLczJ1``Dy}D19_-OgZj~{S+ zwE6!PSl`O@&D_i9AwDUJ=gofYeap}2#5=3RMeb7-@V=%_hi}{8wVb^}Dl4mQZ_UP^ zIsB@nqr&+pHb>L)<8moqG2Tkp6m%#nwSOl9^M}twwPBbCwI)T#*6|dNNw>MLMZgYowF@}l!!Qp3JUM*qbA$z>(u)7(7R znjZhL4(avF(H!`;JqVc6dtR2oqrnf3j+{;=Rt1`$QyTTgf;g7m%Zxcw20X;cGUs`3 zYypGmffvQa#gV3{xOBsEtnUU}*VaDVI&V;jyC>QBpb_Rg&4O3jkq=vRdp8TIHS3fy z!zm*SxIuPK-Yc2M&pW4?SN$+EH%1zjtM-6FOhjI(PRnNyZNWa zH#jr|8vDjNJM;2hTNCG4jAq-C?eBub3?>n3qKsMQY7aw}do1X6cX((82M57feBB60s$P zvG`Fy<7~Cf`uqcP)?>JQ|DwD-{d4Vd#Z4riips<3q!+< zELN+Z@5wz6d!UMrM8LNvx>%1}>ulodD{+7SZbW(`?!&yKB(40@rBK`ue+I#=?d{4H zkW)l&I$H2J;#4Auh>(z4W&S@X&oQ17(cExaUsWYF-U4}~#hKCK2_ZZ8#hG50fX-9A zpbU+o^C`W9^&!PV*P1Icr*onlm+SH|&)aQiXj(c9=2v<;<9S=hJU3luaCmjT3%cjU zht)@SOOP>0MZn$V33%N9eeoPxa6ce#IjlN2Zm63~C_5abes-@dQw^4pz4$ry$N!=< zC-y{!^rNttDx6L;FLidznAF={wH6@WbHw!CR+HYIb5%9D6C=TN(^(M0#ih!J8VL3_ zF2J4zILrLJKBYF|KGXgN#Kaag>xQBC!o$O_>Q8n>!^lvHxpcqXBNkaU@90liaV}7L ztwnmnscz9ZyeS}Lb9J_F*>GMn*^7nJ#(2VT^%n~O8D9x*Y@*?n7PD6;SLeFufPt8) zvexTyEwODsiN3x(*$wdv;g*<;WtnYt2J*!a}v8BsidHf~O)EPr9h6zQ-f_k(ZWqoe6rStl)K z?z9K*au|z5^F41gy7;-8cu~3$X+pM4Pa9~8zhwXzRkzcmFfuxP8+O#HlnmQZmdlH~ zLiXzsE~LehG9s73vQtd?9o^cauF2K^M&-rN|Aop2G|rUUx;9VzLgl@DK54N3Lgm*h zZf^6~Wfih=YxbLX33n2?L@n@Vfih5)HkAMW2EkuYi&3XRM>W^l{4O3vQ;r#*kN{eqr#Ytu0j_KG&{!`U z$On(2-$p^}#gvpr*)1Disq~Ds3tzb&kBq$}eW~f%!wAZcpUEr<08r_QjAc5;C)MuO|$AQuR5%h!qTU?o=@m#f;(*ltEJc6}5zUM?y1_qcr5 z2#a^3Q;TnIzIHi9eHbRFC9sJ~4icH$@X0D40|E=_x^xSqaW|<(2 z>V|DCU1{2-9n!(kq5tfI7$f}d7t&K0<py&?6L5e#HW*7 z5_kH|&27+Nx7d6 zL0xZL53DJ36OZ@yiZBlz9JDrqG6ndkSI#(0`95CZ5p1PQwK*>__?qt*zC6_y@S^#J zfXCtGZX;PdTTD{Y?O<1-!i=G(M0cuSK+Y(zE)_wwX3N$+>3$&`tf1tdPWJrmbB@<_ z@(STBaZzzb9bfJs_B~Mec&7)^;Pp`GuQE-1F?9o!#&q|HF7mob#Fr669^x~+?2gZH zt8?zjsbVYOvJ-I9{&KFy-`%jZXpM_n|6Fb0mxk~a!>dU?P*8r1r1N-z4nZ^D`{IAb z@rtoe&UkKMH3qw#-BiI3d7_Y`v92h2iEr$9-=5IDy82MKkomg!rBKdLa!LeqIPJXK zuL&!S-d9w#*QtZc>3Uj%8Qex#i5W?KE&dPXzyKm>v~k#(Rw*1S)GWZr(+s{E4rxjoRVv5WsC3deqTg-e!LPvLWPztzc@fqxt zSP7|>x7gymIOt9N^5x6+Mn<)v$!_Q7&P0S0+kRcr+aLnMiQJI$%x|w=U6_0jt7q2d zkBKqz`ZyI3vKcwBJxCzgc;-oQTur zjrqxrXz!07r>C~NBZV{L2a&Q{CswZyMU{mAD-OU#l~OD2C2O_rn|4sMf(Vo;N8{V` zRW1KgHQ*=R$4(nF_B3v_ZeuF5ur|(>SLeepIbNoJgm_&&RV043$zXcUJAcVM)#`(r zOXB+v9=P6kV+-2ItE<0maIZgG5SNmYS{NjpqKa&|H5s@CO#+W_ah-=GnLTf>bzyQq z`124n!xUeKaC^|g$DjjXPAXos3n=Ddx{D^c*G`a+fBf^6dBqRcXvxUsmGwZoK zckaBLdhByD3Tm5MZC;R^I%at$tS=b)ICOTI>U-nu>?IJ^QOB82SM7t?)B5AlsYLgk z>Nv{=(2PG~!#o>*Db$?(t3#oPiY&y)#l=`*!+$mei0?RSrD!j7O~0$eQJdI=T8jMW zV5{})8{Of&6cmVoib{w|1+qT@FfKvYdiwOok53#OB@HBX`A$;%DhWBX7iC|l#CWf- z-$g)JCH>ylMFsW}oTkur(X0_=;XxImKwwti)2&9-a(-`WDOp>Zmv>m`i> z$s{rN#GhI!H#STXajEa6WcsWYJ#OKg=5%wbg9%ZLtOL-W>9mI#WbO=ue4_N$));z2 zj;p;F6{rfIz(;abC}5e?-5qY+W;(ebGnSZkd@-Y;h8DCSsc9cUR^6mNdAXQzBg19T=|p z@Y}c85BKW1ozAPy&T6t0J4=lo*p!&nT|~VIt**Khg;;SpI-X9)zwTfH8;AT z1262?cNEp9{VyEV)zw1?mTXxdEO%nxQ2C*C3I291zW+i2!jofv9#+9? z{vwqmQ@T+){Mh9h`@v@CmALtS8i(g6$3o68|O&4xk4{E|DN;e|QArASN^Aq|ngBs3>SVQ*rS<1(T8N zPoMJRn29-E6yOigYJE*Ckg|MQ-vrBD&Zb&DlO9gTz_B^tcg#ag;PET1yo>;3Wt*Oz zv7Mh$9wF5aqhy8WwR9J%7t6*!;rKFq%%~6osd=^|*)b#8R z(ExagUmsIzWcMnl8_Q4pe2m870S$-_Gr`1s{a5a`_z-nRf?*zpfeT00hAO`o?3F}~O$IEtA1x>m)S zps=-&gGpv&1XHDkW-Xyys%(uJQ_^op!eAME`NdgTRaHO0JyyLJ`=(Y1?)(W@qI8;t zCX7wXMqG;+S=tIRHfvuJfd2TBc1wZ!27ajK1Vq>?J9kwyT)TJ%h%JDx^?&@+%Dg4! zmC+<6VoEK6>`%P`UUo@Hiw1g`97SH7Ys)R_a^C|eS2;e>NKM7f_mGRdJ zC`*mGS5+kU0E`3MKaaeLt>;l0!!6xg`w;8r7GM+3DZB6+ss5Y~tuyj?18%w8pIiR^LSY3@ z(w9Egyv>#5DVUW1GLOz>bpFS`+us~7`pYA58t((Z%K-cqIQ5UzLdE08(uyyhHUp<| zZTvr%)RbXX8SI}YI#ys${Nyos48{b1?x|HVgE=7g~hiTzoy5LU`u$*J@+LE z7>A2!0YWXb%YcZ862v8jh4H1-iAY9EmL2@+GXHhkaAABvqq(#UdmU7GTk;nRpvMg& z^%_txe8ZH(zdCq)r$e8vTy>O`2N!28JSwyAM?dbvR?yOa7A_Xkm=u;7l~adIU#SaJ z6W7<4J-?dk;eWh{%)$b^IeLZUvkS@uOj5;}*`X?|VI40;U6`4@Gc#Sq{F%75Kz&6J zRa;A`%%B74pA2T5C@E8m=0A()`-HtyM-)_h8pgGULTUO`C6iLvp##RloA%p$@z;n9 zaDwvgKgN7~32nq~X{d2U;`!x_aDQp|(ZuECl5=%EJo&_2B}UZ50-IU=v9Lnwh=1DQ ziIP@{_B}|ySzX<*hlk02=8Z0c4))l&WVF}6)IUvaf&(<;Dw&O#w(075->7N^zb04o zT3GnwRiZ+Q^a=_(wE-h_$e?w6QHgB3Cn)r1Tmyl{q3s~Pm}JJ&qAl}M0|J8#I;S<$z}ktWGpMGd`NED9FA?#OaOWyeqF5b`ZQ-aXPYc8QuMULCV8X?SstzAuigpv+cB4?0jF zWipA0l$WN>c)4}Rwo5#1-=chf4_>sU2~kbC$>HqP7bI6pbcfR7AxR>%RC6(vl>v=O zYWes^Mpy=h*rKC;fk#mq&$?M0IA*T6ySzsUKXfRpEDWwCHI2G36F850K#b=Z@Vup< zl#G&AMVYy|&ffV;;nNGv{`G=F8db-vku)mppt+PlH=?!W0c(* zZTYI*xI#y9)ZwE=^A0d9n!aa9)q9K=&2Nt7*UBV6d4dPu;0fvg!zxL11tc^Ar`@AV z8oPDNMrxG`JC)@LP;U_>u);f4ED9{DS*yi??f8B>G2jp2>n%?gX%us|zge80$3~Gd8F@dNDHUXj66aAz zjSGg&f5$Rqceq`{F-OoGhgM_A8b%&ccR;PO;+_qzg5RX zBM^D3I)^{fghmdiB!z6dSE^l>A96M0bf_Rd-+T}B+Zv&&8V%4q-#yQl{#iZRe(_p> z!Mi;qX%2H%bpn(0*Ukv=Mq%GY)2yFp>82yOT;@A}4Fc@P=DZJ4o)Ye({uNl1)>H4Y zvhdE%Oe5|WgW+}4(@|P3??4kQRMURMwvUrpx7An8qS|18p$)vZnqCckX?xY5!*~D# z#t&0(^-^tY{1+EXFODC8mOmlG6c)uqdwW?QpL0JxIh)?l7e=pg+lVAMZ=YVOq2wyH ziv~H&*R^zwHA4SoHElP=oZs2kmF=L|kt6I6uB|`S*@bP+7HS7D^BVhhK{ciA!Hs#WrJZ3yh?QiTF0(V1^q=P_-dhI)D_n(FZrdk>$a3<-ClywZoEM0C z*HK|gno~8t-ly1*GR&dOfU{}mbN`loGCETvO2v-cGOno#GO-}V3Cse`p-*%Fu+V_w z;lGu12s%CL&**tTTlb7|F__3ND5zZB^v9Q!$WCAnCnSu1)h92lvT+&CZnq{=&a<4I znb-h4cB%S$ov#_YcpkyT3?}+)`?ySz6$E_gPJ5h5+Ol1afvVr_wB8Z+8923>bO>@? z8Gm4TDh~5CNk3p^Xe^||Sq8gvtPlP|i^IkI?I1$aB%UFX{bVDzaQ3aas`2GXWKTi7 z6BzDXhue+;ULQZcX2iwC1f!|#|OF@Fh70h(M9d<^^wD2M{K%` zwPBXWV7jVzL92eY$URnC!?UP2W^j!$m(MlhA*{fK^RhLHpW1 zAHRCA=m!dFw+llv@Lx=yeDuDAc>rs!?h2H?SX`s z=goZ}yBhG=FHH>pJF-Z#eSzHG$?kia80-MJzfj#0CX#&e@fKg^^;OEv&*Ml?e@kVj z?Bq1U$#HU~2Y~Oxxdd7?NlLBLoAxA+-rmk92z~HG2$QSp>)qO21n}BvLi=ID{%u~7 zV@yY<0V=&CtFC_0(XO^xFD39vOZogI*;FOj3U;_i5aZ1OU^v58E^FS<(gKA>S+c|S zr4ah&fKhT~p%(vXf72=AE*_-npLm8&A0h%0lq8P|6}7xV$IJaiHWifIQ3maXLK6~_ z>*iZ?lm`d+bC_&zgy5JhAkrVh-1crRBTI?hinN!qWZ`YDfVuiFXre<$R6>(Pa_zPQ z17I(HP4w9r#xirU+i`3;N!c2=b`wZ8O#i$#Sw2;@k0GD~Ty`kQs4$wlUWAnz4dqzG zN5y4QkFwRW_#_c$ZA}&*4hS3hHUsb8QjZoS0KL^~dX8y0sy5?5p_Kqy|0j1^p| z+%ek-p1QoetTmr;F}XZRcVRv`#>x+I4t-<0Y0CyR$)jvUHg*F_QuwN@Nn&~CHs9P? z3d~!)?<)vAdO}3xeAfRBwCMtY6<}w;s=2cJnP7u z1zikiK{PhLRAttzvCD@2+}he&USHmsry0b|b~AStzXm$h_Q2}84Ml*e%{pJ^$JWLk zkG~F`Zv9TA+-RXO;zDK=bXKVVy~aj`eU>xoP)Qq5$OnL3lE1ip4RBxObUopCRN81` zEIvJu256g;lds@)ts0P{(P2k26$D7MmX<3qJ6HkD#%AH>s~Ds9{If@ZAL0xll%6YH zwOjsw|B069Z!6VUtj2KT0R69SOUq&G zj@$Lldd|s%pN7-s58YJi4>+|~TgP;MaXR}#?}I495BxA{#a3>TQnTe<1NijQ8<$g` z^Rr)EROcyMr*E9-nGamBuuf!9*!|+mela$^p7#F@V`ILoyY;`m_#0K)`jez(T4KFKYgmy?rE?cWHm>jP1GWUITx!gB{`wx~F?2&ie3M#Ey4T+` zKG2yK3QXT`cU)_dLawT6)p8Xl8%!6S$<<(RQP7X-HO~QWDYYa?Nn`iV=)1=6P{#5M zST7!ct!BSVb!{z_b8~)PhXM#wM3;G+{jsPlYPTyMzv%wp@2e=Vd|=S^QVj!jsWJdt z_>-{j)ynCdr!g^lmlY=1ZS8R{ z!k7@u1;W%>v6Qvrc*N%g-0DCP_{V9?~IUU zVN^az0yvy7eFG&!VpO+dm4rNzKotat+G7pok=Qp8Vg=T8nS_APATfV85|C5D1*@3^ zae7)~#c=aSUQ5-Iml>nxue%1uW-xy%S=Q%qpFZ_C&=-vCFLk69pI4r?7T1{=1hDZF zLQarsV!Ya$0N}_C#$1yf;cNvF$^KF(E%1SFzCM)<3CKM|ZUSUE#uGf%>K>3*)Rw(- z9f;4|pyqhk_ms=&wmvp0@wHZhAWTlf z535;RSD5r*lnMnC7~E{R_i=KYi1$kR;Nxw~REIgg0s5j2z;Qp|;5A4S7Czb5OH9#( zrWP{F;Bpt}Zk&&p##(QjcLTffZ-fF`aRzp^l_8BvYU-oD)%AE0dCG*j3>!=k$Z{ak z5Ot-Se7WKDa)Mzu4h_oTO8#6mRq=-4BG6-8oSoOsI*4=20@*`8qYSdSX8}dGh3kgE zUo4=9CejMQ1{tDS%xYOTz?79wZnr)NuUaBI$4Et`3Zg~Ne`_>=^a&uyFqT@d?tx6C z94H`Y@S_{tFKTM5#e-nhIm(T#Q&UrNRlms@Mv(X&2--KAxo@vZfvb5e{?<(GnA~mu z3-n9SpTq|;W-M#39b}bD^hG^965(SjGV3vYV9K}eA-nn%d75g%k4SPa*Y^&rh~6|^ zy}fDm3zW|+ffq)wp;o+dvd@Jd)A$<9)3KDEIe1cbqv)!wHdZ{vlLcNdU`bb*b0~!5+R=wf&K$oHv)5W8W6w42g$)qHTxz=cGN`q=ey1{JVia#|*J z=M|Ru)feKlg>`%0C9fI@ACax6im=(gUSE0(MVHnPE#@tZGRrNt{m8h4Mt{f^Mq!EF-SRRcY&Cq_lPYFr)#@*^8imsD3#HL0FP%Yh5j*BK7eNl$6x=KV1Cx z$_eTrP47>iUu#+1j2;1x=-bnXDB3dqa#amgRZ7O7DttWb)xaaWY*xQoJY1~h)zzrj zuQppa0f7jh#O~8KDJZX%C1c+l*qQAq6*1M4uVG$)>|pP}>6B$4(LFW3Sajj?i3&oX ze!8RC)+Q*II}hX)d)fuVR;m1SR4=x#YFxgrcjo2ftbRn&0ezw(cfId2QcWS12l@KP zb6s=Ej&2?U>9=w_QEXy{@{UmcX=_ER2B+O#GPtY#y2A}um7i8dw^|;Vb?JZIk_;O) zWR6koJ{%k`mR4$P&ePl`0zl-0i!cvamXrM_S&;t+-NjGc>`UfPM|G15gy;?rT+&g6 zF2xmcjIG>Y%w<0p&9xQuXTH%J5khX1VFDSxQ|&4lFXuZc0JlWqvHPB_t5M>aqj>Wb zNJ#){X;?f+7#xy%G|~FtiINl>qdU7$pt4G1kAZXX;+>l1Qad zW_8z9CyxfrT*iOyves5-2O)V4=rcsb0OE->tiyoLTm6Wi;z7W}Y>K=2jv0kDPs%2l z$zCF8XTFwHo_b~k0GJ}WOS5MENK%RH*OUzD=}%d%B(x&_AiYKSk%_BH({btp#|2~J zqN1t*y3Gw}o)FAd>WZMOPMvk`QR_xDVh~000y39ys2MpWB{dC`$QQi5i-M=+D$Iu7 zm`FSU&q{n+({-Ux7)v!vT-+P)YUI*5=cYio$<*|xWz!%*!8Sp`{9NQe$aGWh@XY$4 zOHELKzW_8?_A8~4S!Vn6a|X9j8kH6~98EF^4QD8SSp|9XU)toSOrEBB^FD+A^qnnV ztQSzcgOXMII}l?a^o2hL2!Yb3W&jLJSR5_;ImX9*H$+G*ydfKY1F9TTgAa&$m|IDC2rWuD zMd0`TQZF(gs_i8##zypB(5*=O4}QZ%2vkoB5g0Tpb})=iw(BF9B2|$N=Hh6!*LAhclpCsK^PQT*te|T6$1z&8IHfg z4<7m)*ZTo_L0T!U^#-NalN225kp7y#T#fkm;-|VgTOT=%e@-4Bo0}f*0ERH+FZH8q zAE;YArD>8I#nBEM{E{hR7$ymtLW2K%UZx&=aOsIu$Z{tp|8qH@zA(VuV_$*i)i%8% z4U6S1-732~qiOnGFUWA%Hws(Bz38!~81{gmXHncdlJht44J`OKl+^Qpqz0nB+9`pA zY<#DEBazI^Sf|sh#IKe90p!@m(J@Mu&X;GM5{ccLwFyO=M5koyM_eqAg-GEB%8=I`gsfEKVY zoU(JzCs9ECA+`(F6DreX+m3Vo5Uw3gpD+_r;nBY7V;T+b8A=SiXq$o(X=O zbs{y7YO}7)i5_hqC&Cd2(Wd?V+bkTZkJ(46$bvmYm2+390-@-5pS@)v$q~O-!=Ku| z(yFW;Z6V)FbH7p54XXvRX)%v-O6TR@ft>uN_1@1sr~7cc;a=YF?x#9FU$b}gqY_e* zQJB!op$$OP-%s24xv!{|sPb)dnr&%$X_!GPZncf5%46TXhY0Vlf3t2@9gks6G)1a+ zwu-6ZL#d*qs17lv_QFj;f((0xpspfgrKTh{1D1JTvWGoU|ge}3DT_rXI9W@6Q`FYd7=cL`j_I@F(HYA3>MvHylieWP8%%+JS}kyku6Dx0tYo z%x~ZOLgR8vG>S@l6UiuYqF8Dw%oAp&l}QjP+Sz~T{9X;HF=i4VG^BEoky!@Zuj-^| zP*kp?m?<2JV+#va3u&q8Sr1~_7SBma_w_56w|HuG_m(JOCtB>6Q3t+M+O?=}(bLpSA2CUCL0 zRZHLf;GZHuRN~-J_^#BCZKl(Y-?}gMetQqi=C_QjsqI@~9RkZkp@Op4)Vl+#iUVeO zu6+|%$a@D0xRUuWc4dO>O!f~bW`2%e;Ide-vcw{41VL2sZ`L%#@=6EsRG6{G@8;m2 zfF2H%cFoHaO6&Aq8WiLr!dc$n-)3yaPcHl*$r-Vp_-f)>gRhL7*VM*aG|pXq8feQa zaZHquYFbQ)j1Eh)OG)LNY}X6&W)6{&t_)8{T*d|eq}8y@Hx1-MGh;^ za?#v)SfqYQkdKd%B?l40jb31GmI(nn?A@u%klEzheIMdhxI+&zM!_R*u>V|HS`vwx z1ZiMwpS}A$F4lo(t9aCp9_G1J8S;~@3HPNRkCA(%X`-kUbL0}FN)kx9{6=O;?Sa3? z_LwJ+lasQn&OC57dxemXHdpz9)bwb|H&}(8Kk7nIaXV;4m&i^NsVBMNUU|bf?1oH` z#&Y9c;`*t8*4vZ-a)OY)=;QHSMcRWyN7s|}frQjag;AV`-|;Q4GX!#@@GM&Q_A(O* z%qg@S7gN7)r?0m^P#8IahKmr)C3)rx=_QJ7MnIvv(-F$YgQ8IT+S46l*uiwwmn6@b zMq{{-9H!WLZlU@>aeWB+?ZxE>+J@=p|Gp)NY4-q1jRE!rk%QP(V&i{ihF|KuDEhJzB@=Nb^C=79{u0fz7zX?f^eb4s6eEcBU-bvS&>> zZ(12vO#$cl+b`pbiOD7{I^)hIEi%*DHK!t@f z+YIJsj~$n#Q&c?yOQW%h6j_31nX#<0a`YN*<&CPQrY2B@V0dF|xp}l5EI-*2aB^yG zPjbr`h`zTSE0OWV;d!YMcbXh!n`!b66$mVK+@35qyary(=>eOc<18@WiFxqes983M zULoBR{CH;^8MnUVB-m|>isOl1?L2y^i!0WH5NsnP-`#)M*Xc+GH+m9V#CNv7A ztq=88Qfqls<#ltgj*l0IBJu0(G`XKXJUX!ng|yQ-Na*isek<2pTHIuNms)N6^KAd< z)^*Xc>!AiPYUY|PkZZYn&vn+$ofz334(d(kU7K`4e~MW)GD@2~+n*R7q$vE-^=cfs zJZvcLaK(>o2RK24tS`@ZMtxjXx7W9|90J*Za}uFcL-tY5NG27v zHazCJakBJ;u?F{!ZN9qe>)y>Jdwbg&H?weZvq?9TiISE`u>>hC6^k#k^^?Jb7!hIe zZX{f#o{ciu6HdO zNRkez9WpBvINB@-WV|#GLp@!4(K2kZyuoO_NVEHL!t1k}bLj*&CwSsj$tnCkLiQV& zj~;=JXR9|%CjgF$yvQY$w6V74biX*N$XBCe-T}}`RKbjd&?4NtC;J`#umg{sos+;P zDX7hs@iY(vfeg8gdf94quHy@Q0iD%eIPWQ-Grhc6=jH}tUbwA(HYM|V(QJ-23gpg| z7?iT8Idg-c9O`92IcccRfOhbiMo-UWv|5L4JWk^;VXF0}tG&_D(Sw8j(7?y>EAGqRf-YtZ3LbUYqBr3D97~+zt}cQ{lsy9VQDRt|uFe#y^u6 z7E&2=$9y+60mBD&e}=bQ)==8ot&%<8A0SO{Bv(nb0{y{*-LXc`kxbyvV|f%^oLc0z zKG>Yfw@F-ob#;kX&qFIUxK0j0!h1Yh#4}A+6%|y?xB{TA0|tQ( zj*cMCeGaHDj$95IT7lMRb2d#_dy1!nW3#x?_an#%Q_5F=j&R~ne%H$QxS?&m(R?5p zKgYSej@--Wilq8}n&M>TX(AiU$gkkYD}<>pfj)Z@mXe|=CAFdw$YdUTx^It+en^iK zxKCDkcvvQfn4J(F!oeVc$pzw@Z#}kQh8Rz0898#)Yz8}k>byKm=uYg!_0T;RCl*>Q z>$5Yz?${4v0#!+8r{FbYqROkEuJyigDrWeqwKbkj-fOM`DzP?QuhyUaDeqMyaZwb{ z@>jSwrs{7OzKi2cX!${a_k%h2T@6_r2IIY@jxdmx8uP|NeHjMQL@+hZrz$KQ(2n+G;kgCc090X#=^7bU6|wV2RYv&ar7|R)wT@L2mbldpm{d!u7%Y zPv=`XPp5~#vjR{pKS9*8T)FD?MLcus331(R!z-EUg9Er^O_*)5ryb9&Z(@935v;iAi=2^QiayS5d zQ1_6SOWiCiD(Mu&BCFi4!S&CTvo5B~l5?j`i0IRfNxqsJ^?eEKaqSL(04o9{n)A9~ z3oZBc(fFdY-G=&@5w^8VB!Xy+%JJ=zz;e1~*;;Nxl!hUznv&AVU;r(HF+)f6fZj^u zspj%wiOF%!Eyw<+@xCIR7NVH9Epn{M@giOK6|~8INI{01Qi)DdRt{Ukmc~VtVz<7c zoj#d+ZrF=cpg{T5y?%3R;v*1-Un)IRUZ0~qnwZeTLOBe40E2y|ZJazF(*hnmRog2K zyu3u9O~`E8u0#bfVrd9C-J$vI3MbBT9ek&eM^F~6V-$KA^5e(ri-X*Rmd~F?rkncu zBf^^)i+n)3bg5w|rKJ?EH4MBn z0g6eRdS{^es(UL_Q2QC%0{^6U9Z*Qf=}FdY{tZj}(=JLMnX zalJium)K2v!faj36NolcUTdGLu6$|Ux50Tu1 z#XwI8>^6^nrg+(|^xE6l(8|`{^?+MiROvoCdQ>O0CtLL$>*oBD7ao8po)-#f^oF&fLPAqs0&s8BFP# zEL?7jtL6Ve)jJ2))otCwK^rx;?KHL<+i7guW@FoGY}>Zc7>#YG;cxeO?)|>|=FgLJ z_TFo*wbz;$bBx`gh`z;2&4CD7)&(Bi!yZ5IyjMW@Sn^v0`OD)ckj1zKj@-d*th(Cd z=N~t(t0SNBN1pN5MfsjowHG>-lzVDXu)06gJ-K!sFJbjihFsCj!(~nJVX9SV2vYe< zJe$9bT5NB3Eop7!x3t{#$23b!Rx~KP=HBaptU1h468PAKQhGhFm{`|#0mZUkN6z6E zBy*@pq8ikLFBi39AyRPHTi!zJ>zVJbr!&`vSjZ;>P-q|}gHw2Gh7Xr(-TdTy?}5tr z5EVN0i<_I>A3s3zerY+aM8G@!?WrRXQzNgz<}s_0ESx)bvRtZ28e1B2yYH?}Jd#YN z)izlHHi};6X}+a=Q-oO=T}4G=*32ql*Tbw|UlUk2yl!cy`5rSv*@;O=0PnrtXQK^h zM^zt!0Vs{#kLL=L%YjznGJqQ^NF?ZSlYN89b+HZ5CjGNSmV3Sy)9ICIj2Il$a#zKD z0jNZD&Q!{^d#9%~FXaY60*g<7+(`~NvtR$DkO7|YIY7G>ihIJl_qTZO1Ze&MtdaQ1 zqhrA_49yn+jn?Ss0o3-k7=GON{-*ciIgaB&Xnpxo1;C%^1BgalwjTbz?0QovOBnG4 zih5Kq{9Ye#mep_Bg*S~$htfcUphE`Gco>h2GFba~ws1NUNAg!__$#OTO+lzWgs@ly zO}Mqk{R!3kMw_u^0DdQMQp$Ee;>Yy@e6Gi9w}&z9URnS{D?oRW-gen!d?sb#E~!xJ z&{lI*a}{}ar%_G$^ULV#T3sE`hD+YVpVLyJ_DSQs%97n<{;2(;?bl6SWq~qf@rFyg zF;ujhTldGdSlkD|>?uRf^U-1S*`Cnxpu@ynqx<=}+?UM;Sr@;}^|H%jVQkp}sHecf zz;OAzJ^>`Dw4k&|x~`zh^Fi+=SEqlH`}84M0!5tQBF+>a+1i_JM<&g$_NtsM^-l+x z^$M|8URxg!?AmvmfQ-iO%NI5svxnR|-*PF*JI8}}Jd_NN$-N6#*;9H8a=8<&X@{7nsTXRFOmH{dpZ?S9?cS zM;F!)qi>Zeo}TUz_>M?|(%C+~jW}5^LpaV~>*`!Le~%T?H`$b{wGVc`zH5bs(tZ7G zt4m;Cm)JO{F)s3*#P4tT6wM-Xw7Ahx9vK-~B70brSNBR??D>dKEMNKnJOuOf@vy!_tW(VBm_iJVd27cj&pcLiz=W> zU2~HM?0bN$AkYIVGyfTL+PI`!y?B2Eh!S0FcI&R^C9LxN5Ur-K^e82( zMrN79$c$P7in=#BwC@+~a{PY^fZf}2$%f#o8X_Vh!(kNrUwxZ}sHL~gs+&?y=aWPZ zZu6Uhk&$qHET0K@=>l{wz~j>rUYMDX;0Qn&kuVw&tcQ^9eniFQgS51q zyq61YUXso9`TroNp|;`*EazWRkBl4Cj$j% zKHfru67ww_881Ygyjj)RtDc8d!m|q*DVnt^W3w%H8=e#04KMeP;OrYSn)wQ!n{OL# zf<2tRghSoZ%qF$9lnnkPyX~VzOMsDXEjp|Z0SF$hS6?{Kf%Tnw0DoS7TvIGnQ{z{eI8oQ)>5 zh55I84q6dp<-HEdd^DcPhpEtOgVEk`;{lz1QpKNrAzc24KvcgG{y@9QsT4}th;p{bSl56|NRe#I^-O1d3hi-FW!Qy>C0TO~*(I_^jh4Q`uZBm=daT^fc7d*qO zt2J4fE6?xths3B|^+N9W!I@)li|+gUHy1$6b*fs$LXWgrXgT=yXIiOII(ebcQWdFW z{yGO}@GEXZHeUq#6KUurC8epUX4Mz5YY_$Jqu4j*%T4KzH~(}s*>3(Wcl|3R-2{Xd zIy%1{T3_#uNykI6fDiJoKH=VN)nTj!srnJ(neysG7s|% zpq;RGT~&0Y4RHTLl7q2&l~?-b}j+Ln+OQgWuxn1PrCrI!%l(wtE8}S z;QrZ`KmaYu%6H^z>4k5FYJuV!)=2@Z-4A;XZDq4e*JGb%6dOCpx;5d3)g8$=sdO?6 zq>4pMsFU1uS9E@5+Xv0P;x;1Ow>qZFEqL(rH**)?Pmg^?)$Zi{pySJqmA|f(-1CUH!zWeU)J6blEELhAM z{>;|Zd-J*T@V2+FEiOI;$dEpsZ(}Ul!};A6>vm$vZo!zxO(#913y=WVTk_v&UL$Wp z4SMy>F95bB7FM#tRT*4dc&nZs z?(mp1oAFszYfUl$!?7;Xn$!opT%6V8k@x+A>Vk!A7LPGiW+XomSQ6VIB(aQ~ zY>F4xAHf%CIk{?FR>k8jqT7mr9g-Ad;+C;zi$BRB8SPKaIXOFsDwP_}fJkW>&OmYu z0oWNN5e$fDKf)^*!s#y$knu9iR$uXqua7y98KX{?6uEoNl%CJV9 zo(~;e04>3ieM*M!SIshZT6JWldt$&#JFy1pmr|HY8WuB2*wV(Ye!KwRF^KZg|3Z}_ z>Xm;3+N;ETrEM#70C>ggDi=_&Ye0}eG+nAvr)gWI#Rwx5@FLa%=H9U0M8l%0?WC;i z*#BtQ7XZ>LGRsp=z3IpBVsZnxp;_(sar$m?8ONio-d>^LP+$6nat{P`YF#{Gr@O~f zpl$dHsXsV_LPE2x!a#cZdUsfaW#??i>uRg_{&aD{1I3xgY&ZrF<%2qD4B$q%+-#m! zTq$?Wwh$5m1P$SFIcAFGvaffmp`fAP{!rXDXmdB&Vh|_+PLU>C)7Kp0m5bT(-LU8w zc2foKm%GLZbCc4Jn{XPk+sk_76E%P1|0@|w!<{_6PF7_+noWIwveAC9V))Tj$*G{oDKi_BKZ8a7we=Q8lm7;k?ao zci0&{2q34Mn4-G5b;e`ud)-+&zAK)}z8v4&yl>Kc4imJuHwQ4|e^OMGd2ulJ-Uo}> z!}{j-L%N?FLIQ6`$pD&o4RM6Dp;Ci$m~lz8zQ0tsQc8jQreLa?%U7C zNf?|-m%VG{J16)*@c}A__W4YJDBGB*4(PRy(CSqLX#H(Uy?n|_m1xix>|7mnueQa; zlW8#&V&@UtUD8}x_uEdXP1{eFYj^3Te9+plnT$$|?uqDWbTbdqEvwG~4%)D$0w!)G z#-8^#PmygvZlDrZfzf8&t@eZeL+fOJ|NG+xw+BmAfiMM3_50(8px|V74ZxzTTgFsZ zFxO;#(6RUaGHF?Bwb`Xjo!t-I$nXsy5b{*l&7W;?Jd_F_t ztA#2Irff}yAEzgi?b8i3lB^H#0(}7Wc}xf2B0nt?taMX=n0Y_FENL0B%GRhvcx}1i zXPo8nyLG>#1M-^RsrM(aNV>G0%8hA5@`yL+yPIZmZ+FUFzh$`Wkam48U$Br;2cwJY z51Gd2GkEta3dP}nI|^RX$y#ozY*pHS?1n!$TyeusOg>P2t26~5J<0%l?03hymK=R` z4BtB`Ir-49U(6-e@*W-@-@biYm;{hvxLj^EnvWL-;P%JSc!z>7zHAj3ett(cfMAEU zU>{YjjEhOrd?uhNE}O|PO+eEjP>3+WCRO!LYhewV$3Bf6SZZ#z#huRx8^@ca`PUt`62)7i>g?g_-fQ?15NL;)Bk~ z7Z4T!SpT%Z&R}^-Kzih^=;85=hkHo%WbEuyidX6%@5ZaEJiHE|2nbZACPxF$nh&dO zYrT6NTD@j!9Qj@A6F;8zeUa_Lcpi7c)tQ6gu+|j-{OEziRU|(k#QGl6iJT6wkN$}0 zY05UvcuTM^m1VbHZD_UaqT*Ts2&;xB44d44yMe+OXJdoJ-ygQ;wqSv{(N6_`Y znH`1*a2M5}LEz-4_Z>f&%omM8-!}%hB|0fdc;DV%ilzaob|3E-I9v{SYF(S>n0Doa~Ar_50I$nu6S_@FZ z0jfYs&#teMWSYDl&Oka)18HrY|INpiz}6m_73h}pJpXvFK4Am6ds}zLXaHj=H{Jl! zs3|clKS+mKp6wh(*g>P;eHjb)T2AWBA_-QkqjjBZNKfraGFP_Q5@V zphrG^0--w-^d&jwZ7Q^gVM4HZ)A%hkCN%#% zL!x7_+ZNXxxh%DCL`3_P)~-S{PL7SXjbtrZP*B7{1GFQ|2PNO1;EYXA%at!^40Qo7(D>lsexFJ4z28RZb<`k01fNEsb zF{!}X$U2d)!6F#nOgCwChAio%JuVZMIg10en*#qp2wF(RY z$~9ICXnFxj&#D*$h~nz5A=fnOuqjw33O(oD7O_R0V#*TBH0bOGhUE$s(NGafdXiya z?suHY8sXtT;DLgno&6ON8i)o(8#PGJ&)LIuJs!2F-tvll8d_E~bgb`^!`UaIrKsM$ z9-Ti$j`gLuvPw&4a`y0?%$Hl-|0YX{pK9PhXbF7FL3{iVt8)Ha?^ui~DmF?6Y0y<) zf90Ifg-FvPZwnZVll)OxSYVzc*(iN6;(KUy2XAqVfb`d(GAgzr=MzeDOxtuoSMgml z=ntU;?X5r<9=a%3DtxA&^yP`k=~D$3S<2~OZ1g{3vJ*5C;`0_xk1cgt2MEBg*s;FtUeRC;3wp_O<0nG26wnikmd_WKEUV=)dw6nX zc=CL>JBlmKbcBb)a%GKmv7bovkjW2rYbw~|7-L|U$T>uz*()awr42w)uk7Z>22 zG@&v<$ek;ws*-H@6H88z><@1yL@+ECQ^PG(qxJ;L+w&?H^#$Cmg_02PsmEs{|1W?4 zYedQ9jt*zolYuW0x~Ide{V-THJYK z!vu{6-1i)WI1E#XhDNIOPY_PI34cDIv-S5OMq-JCsu9v!T2exfL$hQ=jlj+I_3=JH z46G@Nk;Z%OYI3vD^zr*hZ;d47q;xa0gFcygI({Y3H2|7Amy#BjmLy`ilr%In{B`A4T>c%EHAUQ~ z=)jqq9Yqg~vXw62q5fxB`~)5gX&edQ(xF}(=nTs=Vp#QmjFzg%b@d9t9!A~W|zr)&-7Zn}q+tHf%f+!ySsPQaqWxn+`S=lG$bTm$H%rvSne7gRVq?4q_~foW2Pn?$7mBpS&FCCk z`)*>S28ILo^8sVwXHSWb7Yucm{EzMbvDH$t_*FL0QuA=V~vXllyDq*yvyiuw0WUPf#nIDel3FyEK7?`wZe3i~DuRZpd^ zp61A`lt_*}<|z`9D^UV`-Ry;mF9_rM?Ee0KC9U+1MR2mc``_{ZA6sIr#w7xxZE10Q z%)DW<9h8T|*<34qMiKvBBV?AG<@eBYaFUXQA1YYbVKYEh`_KLVdG@~#qAAdep#CR1 z^{^He>)Bk<#Kgoxa_nxNbm*E;-^00$2=4Da&4y9J-knilQk6z0#$=~QbaXd!7Hy;c z82|t4YUWgWz>9YFpCMs+oKsB}?H2~^7Y-g##bu!{G0XiO(oxb(GPRJuhr5ppHCa%R zX~A}7Z&qNy+`}a1kW#pVy?#CTC07t$^;;_Bqb#QCSOQ0eo5rWq(c@vW>c{6 zFoFL&hq01rz?q}c<|ajDvyX9zQ1OeMZ4ubu&@m1xW3-SSeW|Dp{_qHCioDV~3U6U% zLP-nSBSI$Q&O0DSuJmgTR9XpUmXK}jeckQR7mgnet8NQ)V+;#wE-(pm%Rv=Svwd+e2reycZ;9T*)S^C`1mW4O80jv1X1+M@@_R20QT?SU4Rn-?B-(^%)>W@7Blj4F3Kj+GDMAHO%Nj$|7R1eb zD9$aN9X7!#l_aFKKC9t<;_Vmckaz#sWp=3sYDODz>+-mdFP5jh{gHmad2;gXS$dAMCKJaUup2l!P4+X6vk`HPJ8-3~vry=u?e#R*g~-?x z2Lntb2e2qohW)&`dgX37`6co|Per!Uk=q~?oU;rdi7ULvG^na`;~*4^DhXWAB72EPj;{11 zDx+gHI--#MB`uG$)-8g=;tNt#@Uf~bLBx^>I?S5 zv+Bml5l%YbXDDR_70s(eJjEdn3wZ=15@IF$1=Nl4Q_`@U09V?xlUwZ1;@-cLAh?da zqaDz3nrItEqQCD|`N}0$2;u>wrSMR!1qp%@M>hz=s9b$_ypa5 zT$utzAD{R6Jq|0#SvQ2>cfy}vusqkI|iJ?jPu^X zuJ+cD+Fj*FNAJo&p0fq>71Ln11aH_7@*8+z^^FO@uY=ngnXm}YR1AaAggtMvrcKh^n~~YSCzG~E;Wk=Jf@-j$U)mQM4J&$ z^vFKlC7v;Z4(1^9sise$cdI{V5YbgtBUJc(MNM zQz8LZY_2MqyaIvk^t*vvT;VN&8w^)3lyyco*$~^R6oq{VF;Q6~@brtFUv%^iQDwT3 zVhy|yn;C4{eDfkgBT zT}od?e&SGh{iM7zC1=Zx9{V$eU4W{6HWa&g77evhyqLMk#C+0eE6uu$F_g__5#%XHUX&rJ*jpNh3< z%VxT8hB8dS!1=+k_x+q!%r7)r&C{!cdA?wF(igvAS<;rHF8(l>DT^E}5$e@r#`_V0 zcI0zd<-LC2o}BkxYjs`UjYCi8$=s3-yVgd(oFJQJAjN>>FV5T5_n1S+`AqYr>8Fse zG}=U!jpIo_HHCuRvc`e-i`Oa1O8jMod`Np15{0QZu_0b@WWu+zx2Et&UR;)Bw5!Vs z$|3^ce(6u*sXv~q#S!4U(?Ym^^|XTe$spZtGcc0nrv8vKmN&-^Ga&TP?T--X#fJwA zKoC0^J*G}dkPjn^A6RwH!LLRE?45`k?#UL{f?&n|ul);h&%FgdD@RbCT&s159cl!- zP$JEo8G|ElX$FKhH>l(`&Sc`tNN>cS5lhBvgRNA`RL{yl55`(o)MpN7 z1}JGf+z|miyPB?_x#!54h^?w*usWGWvahKvBwwbUo;l-OYLLxS%EQ%PM~0>dM>z1| z`OfV6ION%|=~Ot(7#S`j#Mq7C{ZifR8gg%vP(HVd%(X8Wc9d(sc%y!_Z=t;N!93NB z3&aq?mS0Y$v-vrnHBE&rPf zFlICIV*H`vDze}SQZ@xUTRsqye_|H~USbKlKlf>DH5Z8zJFryoLU)N2 zzl*%hlG3fV?p^TCw(2T+QC|VX=LIp(02{o$vv5K^IBPaWUw|r8)1Z&~OU-(ohBse( z+4?;#{={Zk8?y!eV?_m~;Nu0*yM zNwSL(ghQo{5JOwsC3uXJ!BH{k+dy-(!Yr;Uw#nOZp4NaAX0jM}GMi-PX2X+7@DId# zO~b!^- zd7F~`-gak2J%sWw6@E>D?(7g5MNFoRALj_6se+g6L)$e^`LHBotWJ!iezvQzx%0>= z%fdCS1j0sj6u-dG0dK-+=6Lb{EO~O_9qMSeLF?}=!3OcmytGy-t|^FU++iFC?SDYl z@qA&yT``R-Fc0x7_O$_>cEl($MFC+!d!vgk{*7K1)b@Jm-W_+;*cv7H-c7OE zwc}pITrE92NgmvFe5drXgjY&6hZJ7szGFBH$Rx*#eMPOgxT_yGfi=YOTTy|l`&00e zknJm|w$R^{VLhZ9_qUP*oTBBZAd|%U5OdZW^H2hbN-{%{_i>FCA(Fca%jRxm6NPE6Y>Km)6;P7%L1_Y^r}9E}@iVM$eo zYkwdrsopY@alsQUKW3p#fqxp?w%~;BBmLx=t?p+8Oo!V&B*tlxnG^7OKZHjLwOGiw znU zAHXJb>n=;FS2Dugc1pX&#Om<>E#cPTZ-;D=-y1l*)Sz}CuwOP5kq2GcpsG#%s4GbO zsBIv2Ujt-B&^}9a`MLz&%t+vK<>wy-Vz!Y{Kc2E@!`~%%$;Ll(>UfI;u)4-m)gxmaxME{7kXHXh#$>%O7D$oay3r&dwEs@w-W=s~D5K4bwUPGPMuqXPL7EPa zx*{Vu=?H|dgE=y3Z$OqyVOpH-KdyjdD5;m1V_Sk%_YEs)s59@Z7fldld!Uo9CCJ71 zvgIz~FXW%}2EQ)pcM*&+)dpnCC~!CfE@QPRV$e(3u@A{?z~_|5BBbLbAL+a0{n}2F zoxvWLHYDU=bxfU@a|~l8i~xdRkGT3ift`2clzetL$8R(`M z=rl7WDDxEb28i3kg1=^!-Id*p;f!H%DGUVGDQR|~A=Kd3lo>IH@PNbLNFS4BG?rA+{+mdU$+J+7dV|TC9IQ$E7Fm#wTNfS+XvcKv)ztUOFR@%b z)(1s)PzuM8inhZ&NR`|>{k5EKj3X|kyt;(2?dzuC5w0z*%!4t;t;)4y0ak8&B&cUo zsMFrc*@Ei=hI_$w$U~icNR-R{VSQBDt%z36FVv_x@dApSggjz9{ahD38G^jirLm#J zHX}3BA#AKv;{L;VKjWO3{>e6p3soOMqy1P`7_Ctve&vvGW=K>f{@wh`fjfg^v3|F& zMgP-b2$Kt5G$L-RC|}Hwr^0bocU25nE`E?F+B-OwSeJZ7w7gwGQ-~^&Ma=RA`Abls z;hRCbcvX}Zav;f~XZu>da$?IiE?kowS>MN$+3^EmO$d^68XH{Qn_0QK%Zz^+^9I`~H?Vpv$ zEJ$lR?NylT$nA^mBPHFEWX#(A=?$~?Uw(!kF1^EuSUDS^N%a;>cK$3qshyjQ3!!$(YHCE}6Es)g}BU?t8ju6+l4A2(31y7pBZaIhfoRA4U& z;bEd5Dp3J*RRqgMxLJd~;)bH?Ug^*1>#s;*D0brciis~jKt8QCYs1r-vd1w7>dU#7 zEwAhr8T1MaZ4DBE=M(MpGqx)Gg2}@5*KEZ?k;72HLl8t~C}9avqBDWR6mh{a2D7n6 zj}m>d9Hxd*;k2jO+pPb}0!t~E9q8yP3f>}6HmO(EKNbkT%nYCL=OpZ%azS4#gCWA^|gM7MPi@d9Uf_nO!{p(*aw$zL7!)81=TP(e$UHSBu1b(rK62T zyqm0w8MMKJ3_~!lpX(5b~@%Zr`X0!?nTW$>s5)9St$ji?E$t%@Vo;Cj;(A z-Hlyn)cr212(vKZ8{)IIb`05BeS!!MJ6Xy%@8#|AHw2LQQlvmurBy13!yEz>gy3&y z8MAc=h0iK3G~dr0?1s2*zD5)n2;5GDPcfsl1&vMn4-@eNnTCN-Qju2)|KS~Os^kBL zZ8{q$)R9LfvSC-dEtD%u%zKFT=L&x`sX)Tz@M}mEFYT<__;PvTHUqVT#QV^mz+-Gt z=BXrd#2028gI;QprdF|p{ks--_qiM4pkcpf_WubtOMo+MAcK>)7%Ix*5ipclq~!TF z{&{oaBLI_{Niya<$K67l99b`-qs&35Iy7%$9mAZAP(o{c{f$j>9rg65k3pUE)}$gE_(ju_QJ5O zGF)X~^)^(X@L=RR^9jf=ms6#^gGpSec~Mw@{&!El^FW5?j#ti-jQ1VQW~}u=<@WHh zE%eYJpX0N;Of%Yko``-+EJhbbNx|(`zH}%NI*nyh`^8^ z+D@kMP;chQdK|l;p?*!Rk|r8B4bq9gAR-33uW&$ERG5D+)pY}ng6z|8E&>Py*}#O6 z)aJEH@M}Bd=)aCV%08hXwL}Vm*#1C&Z%E;vs(!=*xI9OB2DdSyWCLfXFHmzy6#kq+ zN#)Diq!pQgDYl&G_0W486>Sc1McfrUQ!dtq{2{l5(4e-h5&7aKJeYU;V-sLd>D~$^ zHi4}Knjs)=&hXz#x)hx?c%BixFz?#fsY9Ul;iGH1ekj7jlSf;%r;`5I-Tt;t{laLq zhf9`t@XSkBtT$8Xb9>U2VT`VjaG2SbI4?4J21>PudgYHVp?z7B z%Qy5?vLVRwg>*}9S+Jh1n+f|jCL=R)DS`NM2m5{SmsbDZ&io9G1`=hD`lvoj7YAa! z66AX#oY_T^wk_8GyNG4n0h@{iiHrb znhMK1B|e#$+5Axx!Sj{hJJ^)ku>z<>_MFck$$h-I14NwcS+$}q=I>JozzpJeBXlb)~7b_Jmi)KhKsju~RY1m{eqYrw-Cc&48{ zbKt(FS}=sJI#k%r7?kzq^b+wyok8Ut<{ZX9||?1m0W>-_7-Kkf>HRi&yPI zfIW<;hCr<&*bY~7@h(%`_0}+o@*C-F$jC23n4;P-qMWcAW6T}QGR%Ey@|^gp^%z}H z3Q%y_S8(HLru#HldU6S>VoA=Gxf?h8n-g%x@!iYmZ@BvF0i^ol9Kw{YRErZ1_K9gY z3gEE?2FSTuSq^^t8N*wfRRn!uKHhQV;%wiE#>&C$;Z=>kRv$m35%hiil(&l*j&~zz z1u{rdBY>`2(kNPJ6+Zp2i-{}U5kT-663YL>59y#)e7eZ4ThtUss;S{F6Q+gRhp;Ow zuNuC@2lNqjSppv3u0h#1uuZNq=!NO*PVa0NWWCvdN#X~DS}Q(mywMPi1>%;%GcJ9$ zDfZT=Wa=a;P#M)FKG{rO*Fy=}<+F_#`KK&J1b6;}LK$Hh0{BJGICW8@OZR?$1AbLl|lB9Z-o6_zrRu_Rc zcDVCZl>5Ox{(DOy0$j|XQ)UtSU&B~8##Wz}oIb**5ysUx&kjNPx4yf5cI0&9@jvT~ zJydmS+;o_YJcSr-x9%W)!PD=Oh=fs>AA)YA%t6@wBjRO^P@cSLxW7TzZ7&eV0+cM+ zkd}3bB{)dBZkwY2vsOn@Wu)R|@Hb(IYu|lDNzWCMev0MJb|)ElqOLnMjsRs0&|%Bw z|Kmg8vPm{C7a_6E-O?Gh+O9dU!Cu7jyu`e!oQ9Y60UeEgGo`l*6-AS4Vj#LoR(X-UkF^^^*3 zNLV8)Y~szb&3BMuLPOrJq)@+kKERkpq*pdTJO@OyE?t>M$}nu@yNMc)DXhSBDP`(& zwr*|t!S+Ce-@rhf{&};fvFEZO17{46&!hkqmg^8iqk%)m(V8F51hu+UHBOUE|}3n{T8_JXA^f)SU#j zK7Mm4y~iJa4Pak%6g_&_c~Pa;i;%a}6F??#2(vB*r>K}@cLr9&^`q-W8QYk_qARoR zQl}F^aHmDo9e2O`t`W6L6daupDV!}qU+eQxOZ3;#!BLxj6e_~9=K#r3%=^f>-=sTc zyO{eEd*U0Gvi2uw@-fIpZeVMc-jG^GmY zOIn~Yji9M{3;))6_}rkTgayJUb=PQKIjyr_Aa!u157Wu8brFV}J;jCzN?FT$7^Q?2 zM)7?%U#Y+e9vaD>nDBAuP_9v;Djtk*EAlf}ZGHtZ7#*<^#s@MZb2>|n{-=~J7spB- z=IvE%wwcYv7@_oV`pLyw1|b=j1p#oiCk_awL^HWQ?k~)5E}R`0;Jnz#Vui%pe=MmF zo{?VOQs&}q`^pA2P61S61Twu_g91NOwOOa%Ii_W~Vi_HU_B-8jdi|0J`6lSGLeHg; zhf!1>aJbqR8a$D~e|N!g;gZUG;c8!p2cQtKz$ZUco7cQ!q4K(re)7*2yZ(~M%%P8& ziHu+(lP7sgNlI#Iv*>3?Y`8P5QMJT&y(513&6V9t?G zIt$rc&EIZhwUV8vQ^n7U_Nf9Ys}NLRy27ji!)E)(T8`Ozk}9%Uz-KiZ;a=rl7o`#- z0)KyFoOtn~a_@j|((|A(j3XLrYKBstU#)lErou@mgio?dm?+Y=zZq9OA@vyd(E+===G*RTR4OUjadotpUTBi>u4AUMD-f)WM!{E#_|gqFbwQx>lj9)WsC%2I z+5V$NEG3x%U)#eMVsA-1SF2jh_%q(_ov>TaJ{FmI$Ie*@ZCOy2EH=IkXGwCf17Ro) z|8gJt;F?SH?hMX0M!2CK3fA5-dxDcW@po}}kt_%MyMEa;W+GGuj$SUw7hbee-D)=* zOs0lT;dn0{VpChH|9JKqh+#s`qTDHGl*X?p#E+U(OevepxyuTB2{4E+TPw%Jy?mAQkG3#+v%5DqwZrn;IfLKuK9q)})jBXU?GqWz?4@M`88~kR z`_`dvyQLY`fb7S=m zDyLVn5F@P7obRH}cP?K<9-#0sL40N&oiF7*W!txz#H%>?5W;csfpBa89 zeOBQvph#>+WgJqPE7 z0Qn4AQ7>@nu$(U2!LzPmSNY}20Bp9aen*3Y&~Qr)Dvfv)Kd#U(RnyX=!mh87rH34L zU`r!4r&&Y@LMcGoGj$B}b$K=xCi&bh)~PfvjpB9- zq9#2h$IHow#kNf4N=waUXgzV#SCxLvnV6FI%aLFB4p!mX2zW3&hJbcp-Yq+`buPPl zVjb=sG+#K2H91e`=EXa>8-*EZyf8xFxV9Q2@TkXTq%5@^Ud4NpEWP+6c8}kSXlE;( zK%oW9BjttbNVnKxp$wa0;0QqHV>Y~hkNUWT!!W80EJf=E*Ut&W;PtK_bKf>gP=`)=gr(buy8d#s0 zKN(}21!=>7>t-P=&pymjG|s!d^dgr9(xih_2x=k&lbBF@IaLx2cbp%RMW35Xbu2`~Ja z^Ka|gT9#4eujJ_8P3+$;n&Rie&0lh*znX9Qer08K_OS`hzHHDQ${BvfI**_e>0R_a zP1mrV#r{>&;Z~bX{BkQi7sbi>XYw3?q+_V4BukXM&+8N4ByOu65y-WAKfXytL?jm_ zO~ox@U~ADifx)R|m~EvG+Pk>?+<0{#pMX-|)xirUO|s<&#=g3-%E-(MG$G=jH@96Y z4ljkp4kaQU;+B+@ZG2y1;o~aE6cX3BH^8Hb_21 zM?K&0wL(wfN8Q5o2ym;x#(GSIJYvLFqdT2Y^Y}j7=?rI0X4M#f-&(fjcmpG^ARP%0 z-qNcx%Q4;d<70AUWNdU?Q%7U2g6bj7?DIZGas?gz=n6cZoYlEfi<+b611`w&w|>QW zYO@v#3mhDmW;C=EnMj28rTu=g?Qd%t9+hA8oj<9QbUBR5e`#_NF6(eB)vX+ zr*B^5SwUERNn%n`)5XHk!M*PyA@-(5gdFajDc<~O$Y|Na9`K`VSvw4M!juaXfNO;d zKk3dZ?Y2Ke=d&>U1+OiIu{c6?U&$k@fe_w3-{!15}+eJ$DUKY6o;ekr+7Gy`|J`e+w>USvgg**r{88STqW;aO^7ko zg-_a~Pj9k5_@<9fZjnAc!phDv{kt#hKVnctSymtLzka3b_cb^>p6xu%?*~_)CY2qX zq~Y;7fPsjp_I(_U=(17JQC6I_KO&tv0+cDzjz@n}JbJX_dz{o?&PYgu7#;3aRaU|w z@O8VMz6AYIKL96bd0O1mH}Z0^8C$Ux?=aMEEC=m(ssEM0tNqw|3D@@v0^j=Da53E9 zKYsphezaCWL2+{?KKbXLhSOY@X!{BOzPvf00|7SO)@E^lqIT??LW#ANjNG)_>YF^r zIN>0cv6hBPXYsDtQa*Q0t>1A2?@NqTg;M$XnVCDQsPDm{N#SyA^Si)4(rSj&g;sY} z7a1*h#z$zZd@40_xjEs@Jv-i2yT*sWNN+T#@ z`eFI#tlLLlgqN2xF}w7T^OkVmS3R0xveWH0Gft%}(mCs0i~YvyWu~QQ-1@`JocrFu zZwIj_T3vgtm7%^qCz!)2;34pN7w|5tYf$C!w;A8Q_?k$ze>3Hh17uYK~^gZ$AHpiK|cmF07Z{0)- z%f&9q#%4_3Y+EQWrn-HIwNe32Jqc?$RwpEr9cE75yMCnuveKqFM#0qweXwh@et62hQ{JwOjZsy?2f}u zk87U_nXkvBd6#>Q9HZfOwo?TR909M>-8xLtwM6y)^`OY?#kM<6U4^s=3Nl~%g_~as z-@F>v@j5^0$)HEoZGKIx6d(+9hSO z|2Uau+)&8Epk>#Yhe|HB2)igBwxuNy8U4f0CWarJpTY;cUiqGleTF~%hpybL#P8e@ z5~O5xKKCA9IctHlw^KQ|dFriCGKI0Tnb|AvUn{zm0p1|1+CMD;mudyk(c|SAj}u|d z=Nr%L&+;JhOBWFTtfQkKBgtoqo&7a?3)zM&P+LBLQ zv?QTV_meQ6LGR+2fg7bi{GZPR9iP{RR>&&qWtU1#|EC4~l(W_qGPL&dI|EUR_0OS6 z*I`q3a|-kD3UMFacmDd^c-k-Q?y=$b+E7slG*JT87L;%VJS|hFcYhWYJKl#%T0Y?&z6?79T+1q3TCIP&@3UIr*14|z{?ULwSf zot+p3k-s3|ST5eBhB@*l3=4kw23z3YCK%8EG)HPv@=FdmPS49rSx^h0iaxyW6i#EB}#FtoI{&nql#}q*A|3^>@I&cI%FCJuqXCGT2wVC2+02gzx@=vqvWs_WCVY= zNug!C5*%>x7unYvdKVcr*?GAgL)}H!IMy(021f56R6E_H2J-yxw+ON-|2b)u(TAj{>T}Yq^Yfh z;sA80v9SQ}X{gTw+(EN2on^r^1t)2V8@4wGf}E&GGi`i=Vi+P_Omp1~112Tu=(k+sV9A9xlV! zxz(39JXpI0{i?0`lKNL1m_b6!(PKimE+3z;v+fUZVDGNaRlUAZo*Y{`)>QPMZY4#d)uyeN1>~u>;1fCv6O*~TLV?W!3O(@;@q8!x3${^-0ilTd^*E+ z9EyzjW+%xL`It%#6D}Q-LMssi*0PtzRqSs`|DvP_39+T5V;}W@7vr+ik6j{m+90)O z9DdUv`v$eFx~IPkM=s-9%pYcI)SW>t!M9G`>HCjjCT%X*a3J*- zKHaY<7KhUPcg z-)WM*`wBY+`gH4_Y8T-~Q%-yjLyyx(QXZ+VJLl#RW#?=*@uIvD;I`CI7G*TF+}ZRv zXnngaRuWnC>ZDIdDyOJ}n_|Fs+>7uuI-i|ve8yo9YI&I2;^A8wR?KRx>uB)2&FrqM ztlSnF-``n7czmjmebMmxW;-4AFzG(ax`F6Ml!Y9J{m0k|B$`IJjp*wT>ybSc_=@CM!%IUQZz$QziF zx9F}UNA`j{XJ_#r{f9;!fZ_pg3ZH7D6M7H^TWW%E9xM)G{^MX)kW(P1_KOf;6WzV} z?J?c-480Z?R{EkU^!eV`-P?Mz7k_`gk(R>|dP~mmB-l?M6@6em;;OE4p^M4JaXL|Du1u>l-v`5c zIgqp=I~#jl0`x_OD`@e0Z(}S74Bos~ZaQ~lbaB4wzCYZwU>yl6i--^I!y}lOe0}bo zR?O41FGio`xwH3OoV|TBncF2CpB_KI;AHl*23|1G3_a4tXia61#Paj;p%{t`>P)@d z$6B%#&txQIX2N!Kn%kmV2<*`hJsqPG{$7?^Q#t`_2?^^F^L1^i8hl#1sKasak$t?> zefl?vs28*tgGm3*;vtZIP|fwMp+QQpt0tAf!71&Q{xQ(s@5gW85O_E8$=xS*P#uAz z7=b>s_~1u6?-wQ#?i*d)pl?jQM)dyub{`V|KSbI_HpRZOwEW+{G0;-ga51O%)giWp z+C}O2Qsd(y^U1&E1~G!mg8dv%x$fc=&_SadRH?KQ=Yw)TV6W)ELDk(;< zwLtf0cAkSIOKiw9E`R+^eQB=ptF?eESgLENZPv7R=9k;syOUAgPlWOFFDwqKxK?H8 z3SnSG5)*@K(LL<_+nd^~c{xRQuY|h?m61ccGgonT(Ch2f?(APUxo>C+z1hxC%WcS_ z##p!#xUcHEucz7EtMM&2nfrS%dU^{ir-;v*Hac#4uJul0A+OMpeqksjPL1$J_ig{h z)Vl-0g2s%ToCvO{(ejm!=f3Iq-k{f#$?Rc7GYd0)Z!o`r>d_HtyB@Ttxaej74wf6; z5G@DM{Y^4%we0diSoqF7VDpUJ$kM`sTFAW$SZ84c*p}K%l(};VSLvM+)U0a&GPBW| zdYfR<-i9pX)oXK-vs&=?kGMEgS66OyJksqfh=xs_KpW6#ncx>@5c8=*Y^JPtYS>=-IuIV_EDtJKNpO}T+)Wv#`?xo zMeAa1mYea%D?`*>5BC=l@Ae7jJ^Y`cq(jkfFy2LyuL{A-SuRFSk^B7iYbr(DQJe7Q zehEM_DY?Jb-GWC7+Vbw8FDB}$TSa*l$aq<3v_!PH)e9L}s>rjzZQOCmlAU#24C1G{ zI7jb`Il`0Vi|6PlD?938^g_66BVe}mt%@@Dt;r1~>$30gkD|4a9!#!;W@Jnxyr6++Y|4y*h##311 zd@?+;6A5>uqWQAy9c0Ul+H;})Zp(?HaU`O4Rop)Y|L^WpL`K`Xai@Xh)_I&!wa%r2 z%%S6)HM!m>vqvQ>TQnvX)P23#Bp8T_R#;fvbS9|YTEWG^5n$_z1_sWl+x)3>alR|) zVy+ly6%-ZRSoOlGH_Es`ncB64kJgMj{mxE8*X{n-tPA*|+`OgU_X^r9DX}!)-h@op= zzlDg^Tk9FRK@M;xpK`*((Z8z@QXkZ!P+HtouC9j`B&H{eMsh(mK&Jg6A`>>*YAUT2 zO|^tbm)9;lEhvC5j(&b2R)ll?QBeySEq6V&J2O}NMMWj+aXEGTTvw$h=L>#>^S5+G z`=uw!A#P$VgKiw;R5NYht*XbA@zc|YTEJ<)7xb0OQ>2cbBU5Vt`EaPfiWy}k z3o~18dS5XnyfqnQWG0WUPZcMjCO_`*NUUah%ZphRvDp3cmMb!s7uo;5qp7ylTgwy$ zV{+-Db5)#mT(_bh{tg|hu&ynB!lrgu_bENOq*VW~BQzsaWPD1YKdPoSv%nAJFiqA} zFD@5s#K=p;!uxV!0=8Pkh*lJR|6i;scctO#x-zwiDu}Ch zlp`Ep47hqn$C%5Nd3B8X^A~gz6EnleES&rmQ{=OZ1Pnc0--26~bmVW9WS7(Cqb(&E z4bn0PT?y!mk{Fr`byCu^>tPo^^f?@BC2yZ@M*34j{A{6s3KgEsjz?r&j%423C7f{+ ziIa*TmZ12*uT*;IKP~w;h|PBw<~f|Q2c7i|ow_R)n(>Jrq@(+ z8>jfL<75&qO?7qA{N@0kB*PA?_f@jHv>_ai%AGZE(Q0CM?WiM8p(ko-lj_F3o~0E*VQ{qhoVk>YsjJAj})fkMXqcd%|Owwtf8o+ zt;(}AI6Jnq^i_dhLKf=v7nP*ltH=ID>h0fhwwWn`X&Y!tTN(w(0SvSC=F3fqx%#KoSXRneaj^6p7 z>zqGBm8&;8BB3L14bg}_PQR#Mx$q=@Bm2>w$wbEHepd`BUyVVT9~c<6qae$4Z`MU& zuJUf_*ik?JI@rgm9jFH0>@7lF&x4Cl#K}nj=+B(xuvlk08kXIm-x8xNOQ`Cn-~J$R z#hG0DYU3Wb)4my|Xu*z%=3T3(>Z`FCa$eQw{fs--R7G6DFbyH1j)*Ym!>hLIl9HF#Z}a_m8AZb^5_0dAYh0vDFuR}Q6_Aj2 zh?D(*gv_IvTG&r2EAIWXORD(zKbBK3=Kz0uvP4plV&})$^W}ikd#jU&KoE|-qEZZi z;I%K==!&m64?nyX< z_b|p^L=;y;__>;`+4UWgmdaE{lHCCD#7yKmjRjZxrWq*@xWD{KA{b|AY@p7{&i22$ zXAga+l;efJE@E$6vNbtPzBcP*=IdKneOGHPU#$HbD~2R%UHk$_zMJ6LSF5JQ-f=01 z$uI5&&mcB$-JN1`z130R+K~2f-;dEjunup~TMkFVV zvS#hc@+<=fYEI+1=u*=iTHvV~I>+#%lAGI0E5Dewo?fTV_0y}RX!~&n=aBV$w7FWj z^`yRccU~z;K_q@be;T>xZuX{`C73g=w7mTMX7G+P&z6#sylQjZ8+sL%Mrl^?Cp|kV z#Q*Vm6}}DEPc-hpqB8EywgVTow!v(mU+`{l&~1I{rc7$41_mt}m^~oy(x`%a(#PhQ zjHC0G<&_Uvj#V?Xap_0^sdU!^Vb1n@|<^=}E4<|bpFZ?+6Xux4ner=0k-+#s0YGi&1%t?qRd6|m{N=*@k}Cjn*HECO z8XP7tv>fdEfW7b~Aomc9`61EE3)$Q}IV?6elY-iR<+`P%VF%ux#btZZc9$03jl9%O zr`{@5hAA4pJ$Eqyg7VR6 z*U>Q%n4RI-^n^5P{x-S+IAY-dezR2A&?#lb?<1ICd|Z6|`X;*`kE+wS^TdMay110T z<{<6wz`0aBy~t~Fc4CMwfn_dR%*|RdPF2U2 zhmWdul!nN}UM3K*uf^!F7Gln$HP9Le>iY%+m1$~f^3fA;VMK&IwNd2@_5GniVN92d z8~F;<2z_aVM_(7Ft@i@o_iI-K_sy7=unl%@~kz>M&m#9?2-W8L&FH^aaBjAJF zpW)Y;&8pOIDy`Vj?;U0cnr@1v9oDm(xCES1yJIJB7!T15H*JVTdiY-6pqHaXPZ1KN zhO-*$nDfdS+Z&3>ZaqJLK5)aP>fU-@NJ#gUr>7Kkv$D4@t}em1i8>luDb}p5Xn5(F zMv|E<#tly2+8;z#ud!tB8oR-17cOR0jzoF#;rbmoeFlGC`(FXwQb7ZS2NFJYA}419 zo;0C>aXB5~*JQ#JgLPH4`xhhEJBAD#9qV%oT!LIzd(&t0^YfjkN~eOJe$){b3v22?yc=Qc2>R0kV_W_lWUb!6TZQ+zojKHI(p$h$8thSIJ$QA!w@%4dZu~8 zIY%N_^4vDJ`vu}DoeEkKEG+jUZA=71?_I91Nfn!Kv}=>^S<4O~Cqtj#6#rJ{F|NGu zl^>5=#-WjqjWJ0fyNf#0?x+jRRDYaTmN4 zncW&_T`Ncow3@z3g(M~9h9BwJVv_Z)i#i?;kH)U+XD64;M7)cuYLf#CIxF1g-`Ryk zG?W`)rR@S{KUUfHk12?XkNi_^X_pjgCTOyv29{H!j~U z!e(G)l)Igk`S_qIHhsite2j=NdB+U287c78OKUmZ_YG;@-#cQ<3?zoYfPwy1T=>#Y zc8u9(Dy62LQt<&kaD%HwNb#=iF|#wQiIQe&B)Lrrv1`v?7fc_Jf2UbC8WEmk)z@{j zbabR?DVG}5htslQ6Q$!M;|ER84E@4#r>#=Z&WVp0>MtuBxn!y+06r?MZgK!iSCH#Q z2H~ocK)G?2K&hP53njJJ$f#50z$eC>mDQZs*as;&xk%rq-9?Mx2E7Yo8q*@B zuiS49SC@p-XskY%dMF^|`mxsl?e59W#6F@TEED(w|BC+Ry2 z;#-rQ-aKQRR7+~N4)II%$dKMBO4RoMXR%!<--A zZ*E#tC*Q{zA^rcf&LMAcU&2gL(Qm?4u6KW-ellF2FK)rGUiWHoFV~Y8kJpXIi~ihK zjuS`t9xt`ky#Qd)d+*mY?rEyNKMN%Rua-^JxGy8SmQ}{t-H&^BVL|>#nLOE&t`IIq258TR>%vB=lT7R;5swzssDC1aYVVVdK(jl2+a!Pi=9=KM$tE zVL>U1z5%N>i@nw-VkKJAa8%vYLT^|VS zv0Gz)&eWIy7NJx7(FS&)e#B|#4ggT$OHb8!L)o` z{Hgn)8eX7<89_PNSw*}Y-*JQooq+Bf1oiHGyJ+x2L;h1*xo(g&5v`67%c2B!fBEKP zs_V8-qdRp;UegcNbiJI$$y+!I8Tp<-uqgz6u8)b0OKiEwVtbjA+!uVKqW}Yu-D`2i z7VlZ2GlGg#+Nsw4Ad8B>MPEQj$wXAtlzrXMSXz>JS0PlOFUi3MCp5;3-rfJUik?-b zvTujD)Ql6>U}l38aEfA1c)PJqz6sxeaD%>pTK~=Y2IHD^vHJJ->K>0t*D~k$hGww>mE1@VjfxC zwoUo02HxPsG`c>`C94~C{aw!JfZMU|T8Ml!AGXQk+u2gMEL_{Fr>s06W)&;rgwX-Z zgoRe1SRnU2ipjXa2d-%&rbZn-!Vr7$noaL7B`E#B6<@hUcgg8D2;7??pn^0c5Zj_0?#sCLXYi3o4X!Euu%e^a zLCv{kWgX}H+)NvJWsA|D768AQpwr#bY>wX3!3#K{H+XmN(f(UI!$6I#F zA)1PjV@-^3oiMG!dr`l@#M&|}jl{}MhH{%sWwPamB~z$eDR4lrn0XaS>GixtKVkF? zP7F(7|9Uhys@FDrPWsuo>-pY8o)_?Ak(NRxvK2x7r{c#iV;lVf@4H~f)vT~El#N0D zfQZq(>$q1en2wIdVqk>^_yu*sTI$Y~LXuLz3Oz7C7kP4Ir2jh0Lq+v6>0=xWFTw4n*yE^A z_qxaZ-kFUNMA#M2=YFFSLa8>gkr&8A$G|}Cxo$4T$Nejy(g#}}-(MQ-e_DW2wA%_& z(0NJyl!u;>3$C_{_A16-m!yC9)bh7$Q zV6>?dS9_!1-irUdwGsznb-0pfmu9A5Eo{{^Cn^K=(l&D2}*I=ZDK;K zKHQ1d)5O6olzLy^Hs$d7#GRFz*8LxyL5BZW#F|EKA7K0jxt@|W6H)WZl(7GjWEs>F zeS1JBQEQtl4Ap}WjQBzX~9ZFJ6jbbS9Bm1_o@n{6S1G)W^wAqj}O7g_itEXox z25F2MK@!;>&z~wkq!xFDSu=xWQd5biJO(*QllA4xJA^%vB)^k*_iu4GZfsA9UnO)B zVila3GycMX7;bJs<}z|fUf_c50G|nQs;eSy^Cf_YF5EBCDYiLoTf&79r0-v?uMh6IlAP`2YvPObBRRcl_8Rz zhB_ws4QV#grlueykLpj<+8e_3XH4~C;(^NqeWq{IVk&@Dmi2F>lKjq=t)_)648z_< z%I>YtTH6LcX;Fl{=$M`?Q$A%yTmovk5j(fPwE?Vs8@#n7g^vH2&wVViIwCEV1oE>k zCl7M&j}a?z8ID-TOsjUVVkBRDW=C=7!(R=0e5!jE5=e1{=>TFU=QBGKWYjJH|%?R%gzYCL6G(5yxZEj2aT%N^ay_@5I)BbprGyn>#rLB?l(|Igv2$i;dMDdn|*Gr)>pJ zU`I#jsthwlxDKO@o3X0sTpW5b#hgJ}DK1#Jdw=GnvP8A3t2S2D z0;?D4>9RO@hXKdFj`y7m?xNTns3l1oM|yY>p@)4?TxSmQ++rO-J$O{>7p9n(-=z5` z`gA6enZ?@oTG(fP*U!9Jw~HQI#|UD2lKhkRR=ypc6sdJq-p*jCQn~#Z#p` zpopTkLXdoV0M~$y%^)=^d*#f8w-&Jz(^rHXGLH?Nwyq{?g`o~#dIEuIPM`Rkl-jnY zU~mCc`rnBABqvfG99aB&;+SJXj}o)=chK^b=;u@IK!g^xtOXxg^G~FDTa26#LeGU& zFE~He(15A$Et@5HMVo3C%#?c|it05^+48BKUUQx?{@Wy(d9i!e-OdUh9y8JS=}yOa{YHyTa=u_tIthQE zQfINBu~%Sn{?6Oh@cO5k=4mSS(;w7ClYgu@y92M>u$G&mBg<~~B#h&{P!*zz6c?+R zgkWC>du>D&`kFN;*!DoZrX@#cMlZ>3%wkn55B4m3c*LOR9E^T3zx`e(N5B^1#dMP6 z2M@v&IAuiq{yPdCn0ZzD|0tt0i;|K`eiea%o9dyrfVjY{fL-GJ!j2j5PEWv(hoy@^ zXXf}X3DN2B{wJuc(^7w8N86zv%s=!-=;}pg0=U$kObV#_oy=2SUItqF=`X=01S5pX z&6qRj_&TGQQ*R@sapj2z79UicY%wwHMmYvT=!po(P=c~h|Q z9kv+6sSW<>$!$K@R@a3B7mut(HXAj)gik4BpF?w~>Ni`1*jdaM8@lfPXmwW97BqZ?it^yE+p-!-_JzL@;ALWD?W-ZCAA%3?y;cz*_Af$% zqgKZGXeIrb5SZxWYymA2$!g<9x0%t(GOCPz7N8p)11YXg()fwL-nmZ8QnTk@T?@nk z4o{h_cJ=GR@w>x`7d`Qo9VA=V(wL**mn2mS#U_dog|Xr93JUeybnYX4eH8SG;ccn7 zE+T2@^`@i0ZpoqY($ahvV$b#*SB+d&a+1T}k;h&(2Q*_BWtjTWM`UIWYlfbdO_Anl?1L zrr_H^eERay55e)^%l(z1l$&Vad~#9Wjw%I{Ejd({Y2ULX>>H^ zcyFEwpkDfzPe%%mKXTh}ks0)7tt$kif@mai)|Z4MZ3=n5#|qrxWbUfnwbQ-d&OVV* zH1Gwd!jO04(hA5tH|iT|J}9u()P+)*uFM?cmu&W62wdGYaN-b=$68ZMCX%M7r>4{PMCo>K@T8YHwV5V}bI0R5 zsPQ?<>FF6nwNxgZUPIpSD^Ipd%kl1=94e)I4Vu z)j6dtv%I(8Mv(fqFF|!m%0*<_QQD=0;WEpMKJt@>pwainuAB|$hVT`k*ur8H6MtfC z(BYq3^{L^p*SX!Yw4?M4ihi?i4vT)1Zd7ZnE2|G*s(9o&-dBC9@tNfkb?9tlB_^Kw zk3%mG8uYt<7wM?E0zpqY4s8u$8t263|J;m&CD!j8y}0Wuk=(agUNW~`q4dgW*O@&N zV=36UNyYm&g59)TwF{&l3xK+WIy>th@b@SBmO8!)oH#p}_Wq!*6gNg@f zg;$fAxPSRCLRjLx$*IP8F(#Hs`|DYWrWxfL>8D9N zvbfVvs8N6t!5c)}=_Z*OjGYU{%L6$gDBWU7H80#O0)CeUD#7%m?XgAjk+NB&d#{lb z6N*QLi>=%kc;$y!UAbQl1Ds;D`N4LkC}+|h3V#py<+DWNyP9t6sgcaGo{G8d#hTT>|j`oUI#x3v_&>Y;$6=dDH)^|zSih< zH?=s4r!0(5N2|;;E-+>km7U%M!HUbcLR$>SfXG8mtiB$(JzES$Ln_edojS$1gVNx? z`{A}dkgj55+D{px!s)l&Ex0svYrh144o@l4A&wN78QQG=ub-M7pVPK`IR~EVZ?`%b zoRWQ@!Gv3g=rFiA!vMg>{V9*%Bf5Tim>CV9S>!o+xtIeAs6>S>(eYi4xQYX(D=w!~ zNk)4=_p?7$%%S6=BzUre=rFsAii%GHKIrt6Cu54lep%09IUKw^2cOTxn;-**UJ8XjpdRRGR%q2ySXQ_cV zj!(SiU_iOgENtt?K_erzjq|)JjYWeKIyQ?ug~fU&^r1CH_8%rZRs1*rN#N>$d!t*d zN|II^d*MKYDY&a*u^X0U4=lkuv3B(f@%Y;TemA39iH1C+ZLm3yfn8Oq5iN$do|Lr6gDhn$OPEkN)bRncxQ@xJU-~5 zraH@+$|uer)B9e&RN2PxExh(gH+z%8X7h^oiJs>~QGer-e33H!hXHc%8kOe!oxV}C z_xuoDkp_I$l%0^mCnOS`oAU2}T0r=6a9J{4^kMB{S%f3<_StQN!7(3$NnL{gznGA^ z+G-#=;W#tk!8m~WSCiHS@nqZ3P$*f(%4mf(5K=}Va=8U<%wf3mVwL{pA|iag@8o16 zg!Rt{n@Y64u1iYz+Ld~W5EZn z?O1SJ$o{fNglW($H6VjhbIADQbi9FNY`=)kCWPnU?J&yX#Fuc75_b{pefO6Ie$9QJ zbWb>fwxgn1QC8+86~NB6oJ-><6dk-)_U+i;W_@XEvu;Kf+3~h|xZA-!DlN)Itkfnl zhKHw4(a0a?`U~80RZ60@NdMmS+q@{M?@5wEnnLfN{qu#`Ua%6^$FoJZ=#mvXbdijd zwsT=?w%)5pwBD0~Xh}^a7|r`prvrJ^BYrH!Gr#1K1Ay0^PyaNbW?bH-`U)x}0e(Z2U)wDeC_!WDB8f-Q%z zrG<4q95Jj%qp_&~L*vfCxu*$O$JsIqKIw*rF9-OV{txR$1qTvmZB1?E*)~#ige4r> z#~!vMi&3OmV@;3ijuu=HVkdvujDokP8Z$?jS@xwToZGccwSPB|Mc`Sye*Ce(XKi!~ zdu1QE#{o+uAQCC;U~MeO5*o3f1?;K&T)Z!#t)JGsaZO()Q3cA{#(Ru$YwJ9G(;kbv zu|Fs^CeWNa^Zf^*%fOjy+mIFul?(Bu4m2s*P2J1>OY9a8pjH;H)u)YotR`EAp5wNQ zX`<^U3$}XY6n$V|@R;#V_(6*k!r@)Hj^3sdD0(EkS-JY$XDirqyi%@eU%kbo>F%q* zZb_V6!+q9jSG`8n9={5$izqPIp?KmMC9AZP!)vwmi(n!)$ZXX_AeOjAM3^l5vNelo z&qwF}f`2Gk!pxn5(l5-}1s0)kgt;)IJ1V&4i)G>s;Gfriv5J1Mm1R9wWD0k{idizh zbaeCMH~Yze8)T+-Ro!A|hWB4)yLXJ0b>AZ=`eSFO0nVPYD^FZj$FDABu;j^a!d=sT z_>Kt=2Y)2=#7gi9nVy_1OE%to7wKuU7%E0)0n^h{qTf0W!zn+fXK85*aO3gAXrs(? zCS&rv8#;sfg(0+JPi;xeUK-9uB;wPy&C*%Q^@%ne4UcQbCF!SuyZtUXL`8Rd z;%61uo;YKIYSE#g)Thv*ZG_QBg*j3lTqLj+JvzF{nu@fr} z@OjI1%k{L~4$ra9jisG=(cOsOvuU!G?ebH8wGSE+-TA#xfmM<8DMwY**!YA_5Dq4~ zpwUU=Sd{D-S2JkClt<{w+4Y9*(EPi84&(?4eKzLzLu*0;h8NBUL(<9K%sLV*dIKPV z)DOX@N5rMpM0oKFM*rwf47^A%aMAMhY<`rxSrZHs%6bZ;fgy}e$hw<)+|d!MRusBH zJ`zlKzrMoS+v@s+S~|*!B&ie+gs6t z+3h#D^1(<`Md3ShXwEO&Jz}DN|0X!s>h1zHwl`m@d0(|jDr#oi*Z#e=isJ!&JLcn$ zT-;pj-1Pn}r|V|7TXTz-lYAJvxAS~j;KNU~*&Uyl8Sp&5|01I(3h}9H>dXDM?L_zI zks`}Bk|#^^Futb?2~;E5^|ZqQ3{BIpZq6;(vBWb9dz1~1l?~~oqN?jk$5iA1Xr{Xv zU`o7WM0i^L3T?mU`^A%Eq#f?$Ya-qZP=IbsT!|#G$QKA9(pEBlBLeZ_<`tx`MC)m!gK&`J|0T$R zk64to40S?ALSq`S`WmYS25%fgV+#XS`|Rnz1%|WWH>zET7Z(3rencAu2T+m27nqmX zK|VT8FmxPzQ*9*bqeIeqmUDouN4PlX;s^nO0x}Xuk7XlHC-}=tMDpnsl8Dfdh@yR7 z5}u}k(9~V^#>am|cF|A^?Yg}UQCiViBJ4>N8$%_Bh^p5%?@~l#drRgq!6^SvF5L7pN)y-eUGlTcj=Q1ATnJYrr+4twy-Hsz~Kc4~P zj4xf5WTH8v0<%BJ8Ove_I^LZ2t=Yo8Os_09EpA9!E7Y5vUdKDD|LmSBar!+c8%+Mw zz$0rPnIiu7kZpwh(BK2M@)k@-H(nMbh%=k%NLP}BfnL9yW0T-4@j>i{uPUaLw? z0hz;45_6kI6P)D8<$qzMmseLNkwvbvr=aTWpm+fR?{RO`)Z1^oEAzhdHEnrl_-CiB zlI?F5V#`-T=aN33a4DxKjIe#59B#w$P^bRRN@s&o*1*I6r;=BMU1 z7FU<&IxM%mI1XmA3ZNXy;;RlqZj+W;$u%x^nz2k9b&*ISL3?Y|XVTy?$zy?^DJ^-< zl&P#lpd2TGTL4H%m2)^HIX8sd&VzHHZp~Hydt(oLp;1k!T5rUYSxAVa;Bahwm>P9T zirulna0Cv%OmnIBj`~;;SUpSBQX(ZJ1?hX`pIYd0(i@5C4DWB-5^Zx~tqWI9G)JF3 z-u8ykDT=Ojk}`h9M~&kR4x$K8Fe{*9GZ~(0!Vjt}tn7T`Q^xpj{m7J!9Bn@bC$rPV zp&+K@@IFb+#(LePzi50>F#|)W+*tWFC-)$_3p$*}DysL3jYl^)e;nciSO0imVQk<* z&D0}+rzcFi{OR_|O-QJN^q{Bk0&=l)YC}$=@g|D3@5sm{7)V2o^53exhx-klSLfJU zyBOUzs6GHdKa**R`;Ot(Tqd>(R%T{GB1e~~+m!3FOBF=>wh#b-TA05`(IYZ*DZn@J zFTU*GQ(B<~3#C>5;gJ!xrWXL%7m;dQ&&(;h*5v2w3V%So`}L0@@dKYzl_gqMxp<14DIMXH1Mt3m@Ol6t`Z2ZLHuuQjCNUbZ)OYvNY2n zmIB1W4uH;b>pU-3qGrYvSxs3PS2RB_UvtEiDcq$f9X`-wIyaM}#sZaZ7E07e`>r-; zp`M|Jy^%*yG`K5(vt%?;T4qxxhUhSc9+R!`Fb>K z!-RZ;jTR+y1p_<3cZLI#Dr^F^Z2VvM_9WP_5_5WGYZy-4n*nY1R#xvPL5@^caBGiz zuC_2xO-=bseUVSu2v>_8K%#}(VZ3|ZNkgTgqVd%00MpZO>vZV?H=nme9Z3v2%wMnn zwHRz|eGtg5l+kL(mih3P9n^>FTt3e-F5JU!K? znHOi{O+M>QJz;8S800&=KQ!(?ch9<~m3+yk=pRVCZ|=(H5@F1R*tJ(ti`@Fcp_Qz- z%867?U_!!Gq|;Lr_G=Tcy7Er^q{_k8&U!uU{r1t3(;egGe9(7OtmCC19|9Lx3x;W7 zSAKEqkN57vO^dQ$uU(`=Q7nq-g|TzTSy&5Ie^MZmgA2^{Fpdy4$V8XAI(dj02kC50 zY)LGCN?W7|sErexprh+^5FKc7WtvPjo@R-h7#L0n`ffIS4uGpmD(G`_XBI4CEFf9F zX2{(?58QMM9q~6Hnu(vVO`Rtc!@LCr3AK7)M_YGwSgM|dD%{s??e@3b(Q4J_BJI{f z+~bBI2DPWF;J~-zS2EnhmY+Nf=Ws%?K<1Dj(-c4RfH#<0EDvUwovzRI}v;-?NJpV%6D~jIP%U)^L1PhRc&m^@T8r0hT!Uy zWXo?iNl7;nS-8?bzAv&|DCuG(!ihNsWfJs;(TA2mPBKJ8u_qlF5g)hnugQOBfhX+| zvxkS=|6}T{gR%1_9~r?(XIxr5luPl#Yk)1}P<_ySuyVTYJC1J@fsq zi-wdAeIzw!-?|E!nMkI=c<~1ubk1$;dCm zr>?dxw60Q9tbVTC1Deq=#kcjg`!m9g{77I}q8y3z8uz9*J%aPm|qgVTI+pUJ{#P`h=1j(GsayEvKNjrsM9 zCc2ydy3dTvzYOnpuzk4gbp^HF?$<1)ecayO9vdp+Z;}o}^xlxZFkfGu$5$K=0b%-z zjvG_~d75IJvzLT~L^KBDqC_!oC;xsCB(kpq56Yk}ayP(9C+u(m0ty=7jWQ>t=jKM49QcOUF8iH+>#i;cl}yli7{dq`PrLD4kc(er zdImH(gMQr2(ih+kg&~qb@>Zk<@TL+I&p&iDXI%ED;n#A#4@&4b+!j^?(d%Az;cb}B zCoye`-TyZWSev7);8PYmoxCI!@wsJwZfOIB<2UQ4iHK6QDH<85HrY?U?*w!u-nb!M zB^tVcKp;1N{2CXaQx}<#C!zdq`Jt|0$KA3`uMzZiM5cq|Wc+cwO4P&W{nTH&hQ|Ai z=WN~HnGA0N-Z>5n5;9KuLe&3R+^Fwz6z?$SLeK|l&+A_xCCm&oy(+-X5b}I?#5O^u zG^b>zq8eSs8QtC52mYPhm7gb}llfLw2d7SJkzD}?LoJebE;?LcAa3630MOM4+oaL+ zrmpkB_4Jp!>Y+79l6h1WLF#W9cG$lny{l2s?yQTf&xL(fV^EPrth@8EgX=F}Uzx4G zUDAn>UHybv?R?%#efW649rW%XT?ZV$%MBM%bLK5UE>L)zVQf($gMq@%7HQI6%ss_7 zeToYA#DeZ^H+?K|isl0-g%I?wr*2YfPhdlll32ZQx%A`s71DQy8Y9F6A zQw{r`T3<(et1Z&I(8%kUhpm+2$+|>gKO5cI@M*2TqbC8)e^4a!@>CL&pxr@%k3WCq zGtge1XG9S1EzI zzMh@sE9n9ZejMcX2ZQ=LHf%a54A5dtp}$nwJ^UR$-F$^l%HMpKZQz7mb)7}uV2{S8LKC=QQQ#vCD`&GL@(C74}Gk(?iUZ`$A$}^e`}1_ z-d%x|iO1ioUczFS;-UO@oHY6hsX~3OsOTm|&&~&YJS0X*afj}ZU}bu6`a_|Cx!E1G zv~r6#@wvZ`evZV`6NTfid%79;mlqdXAk=APrmDJhevhh9@Zc$G;Z00J+Ut2R`_z(m zC(<|Jurn>q3SIha56NJtDbb810wR*%MHe&G zk+HEi{20;nYX>!O)!wol1G}T7_m&f0_w!ti}_7AR-10P!2G^9t$QkHhoCbzQV1tzB2s2Rn!P8%Qh;Gi*|> zVxTEe-}?_Dk*$C9-MSTuRK%4+v0q5~J7{)|g;r zYC+`_qgcrqi*&|hQG2PQ%c(DAr zvj(YB@|0NL5t=qTn{2f!K|b1h{MzLEmjO&TSpAdzcL_Rn&w!76%>i#`e{W!@2Mi#u z&&My8Y7mjC;BCOj?Oc0hT$)7&QR*UG{~|H{HvbUl*CFHJr1D-I^$yItRYT7L3pvr zoaSY|$slsaJMEKX= zubDXcK}BZDR#MfH7GRf;&*iL{>ok!*iZJVgtWR}ZY@MR8-)9qbU|T)f;=Qb3WVCQK zIa+maT_qySFoJXC1Y4{SzW>*=d3Z+~hh(C(2OKVdoAVag;>JH9czgf0B6|5f6UXB9 zepHl1$gN~&v-~zL?(VMXKw0o&^%d6(3PErT&M?;-=%E}FEibw=a;jr(1UwzosWOGt z)wjQn9|Q@x?vMPvnU55MH@aQ#e195b3Zna1g?-mA3!}XQru_W=uV2(})&t;`fLCig z^20{^ed?VRG?bG~7=0iVBxthGJ){!}lRDc21Ugz_dk2U){GB3RFRP~t3p*VT&H;ED zG(lAiqMDD;vrF!~j~{3k1}#bGjxQV$u~)JB3*D=0(?4u4ew1Ewa*6oi1^9qjn!99u zNk0`T83(-j2`#@JI_cTz8A>qu-C9`rZoja7#MshVy(FNbtajgP0tFJfiSB6cKCr?; zXf=}ad(5z;kP&1X=nhFi?{p~$xsaMZSskk||KrYBTRSZy_8usYHqLq=W976xk{Y30 z+|8Btv~)|GY2)7LmowM`gcgALcSVV{(~qx_bzofO{Swo9zOcLar^#8g&u4r}9gl;q z(Lrjl=fc+`a_(TiiH$>Js{2DHvTSVQorErm=ts(Z)KokQGQG(cnfA`RsuaP69)6$G zQ=rcV^U`z#G~WFcKS=2o;QzUxPj3%60YTEB*Y}xZ>y4olgvF!jmzF^MmW^{73p8jB z8X6q@0!uV)MWwj4?)Rq4$V*XBI1N70ph@v|B8czNpJJG5&#%v?=GlJz?$Aj#8fU(| z-0^j$=dEYW(0rO-33+M=e{IOcPTd|J5qXIIKMQSIb`?n}Hc#Ph?E$vOgP@Q6_cLGr z-vXgjil*f4w09zHiJ#RMX5$!OP9+q+tae{jk%XCDrN;wT7UTvGpm||SB8|Im?QHN? z;CQihdyA~~G5iqJ$kd5u4pIW3UO>2{<1_iW=UO4*#4*KR9WtQFHm61CUJu9DO!nOEAL?f-iTa@&y~#XIyas z(wH((g*E^9p+1n%YWQ+KBk%mCN4zI)j1YR-q*fhXL_l}fVc>T2GP#sqt>U@4e7o~t zoa46>F!|)xDi%kj=}j)+NIlQnc)ro@yF2jIoFK7?kdfbE@A(@}5FYx9?}dHNWAT_K z=TtsJ;0$yvb}9)DvO0!6j}8sq&dHm2DK9U_2Mq-FCR>-5 zvh&0vP%sKl(IjROd9(=bP2ANiXldIMd7IPP1~>-zdtA;p)#N^yfhcJuBO$r!@YkN` zRq=@&H-M4vUYUQp;6;2Dib;YlS*~g^fVGkP*$ReA{*1MgL3Mag_%)12Of%ZC+};7# zu`S??sDou*pTpVstzzfT(esd&0&)$rKG|cN_YJ&I{2Q+MOj{dl2ceg?#)9b~fkd@t z2cf%LZjHiU3`*ym!Fj6;lm5H9svUM(d!U#9%lRB5&zy!`>v}2OFoh%aScu20s{tdM z%3caQxA+HLN#}$ERabwxrD|C-F=*PMHy;i8UKF|xarBNFrvD9Z{Z;DaovNiKNMCIz ziJ)J6CkiS;lM$!}9Qqw6eDWEFO=kT$UD&~!!`Adz^xIdU;bwgudf0o8x6q5RN%Xbb zp{GeLCzOC8Wd&fOZKf^_x?-y|ldZkD1_VfyA!6Sp^wSBq3?25otvQRp7ZZdLFesz~ zBZtCE9ph~613V+*RF-O|46VMlpo<~zyN(skhK9Wdix7cA4%;xsF3-une!2;Mxy^~D z^Wlfan{@AN;_RC-C;bIknzu>c3VR~o4zhyfAEVB%mwSDMrFFh%FNxct9LioF`wto1 zXjxHzIN0uhI$w#CH^>rrO}@1(9pN+z)daq^BuLQRlk#PyB@8|J1{p8Btt{khs6FpI zbMrShwtka8{&u%oF-c#yG7$lDTC-uwl#t&Q z@)T%7JA77|;rHF|^nJV%m>1A9K1Ln@CEBB@hpLVD=}jIM4y29Ux33yo0heL z_7mhB~K?r;We(OPX4u)9ubM)O_HHvPPBnCb5BdhC3l)`$1swURUDrfF>;CEPLh`@SOacE zL_4qZOxML39h;e%OZN6m6Y#xz_)ZX}%*YbdL?=j z@$+Ql<*kh;E?XAM$A)}OF8lYJP*~_DyPu}W-WOh9!3KGGS*j`q_2LU02a$EnKT+dP zD_dIzzpIFws_N+t-QI@XBn6OJS{`ir5C54uy)8OGr>)j|AbyWx*DvgFExC4_{;v3(f@1Af|K9id-1ax2NM9Z!DEgcC@4q_3xy+ww zw_F5%hS^xF#Jm}CGDTFT`R!N=L9y-}YF@dVcyb+OI?#3B=fuS^W5dV>4GlNAxp53R z2AvS=6yJpsUxe%yI&@B5Zx21gIS_17b~tolI#O-gBxvdN&Lf}N{Bt(P{E%t#>kDIR z!tpqKe5${$Q2w}PyF5M|M|(GQ@c9$)u9I4}`U;hTt`Tw5z!@#;)+t28Lc>W#WSw?* zX`f=egrlf9DTBu(Vmw-g%^A<+TjnN(2Xe|wPL7<3WHAzd9+xO1Us%xE!vyxKx1E85 z2*y=HVFUv)70ZbqkIH?e^w2o~>Pkf=s*Be=lbC5j6;$<$tL7&?6P>lGmz8OtlHp}z z>2S>1@x0=cQ`^Rb@e}%Z+(q{z6Sbu7(vZGV*#Bk$OY_5L1&GGe^p~W8Co;)WE?{tP z@+@+u)aP|pwbTLH?d#c=K$>2HqezZwA_um=f$FCSSan_ZgFG{)rozqP*aj=(mfpYv za);ex_^$WkDdNA@c*-zP%G9sm?+%DMaM^cT_UE-2?=4uimLOLea?21SOUwp_(-0{5 z$7xdJ^Or1uL2!!h^_Y~R7;TGy`Ou@Jpt85Qt`k;^J<@NT7OtTLWo~0DgSXZ~P zFnFxvM(;@rJ$O&l;(|3Q^o}G7h2NOc8>Z>25PddOnheH2;bVvt_KP(8cU-!Z__-^0eve|BB@tVhUnbqSW%{hyhLnihFr zfSa3x17M69Nt%(BOkyH3AG2}TExyx4mbE#hpadA5xoFNce$O#wBtst(Q8w)8d2EoR z!P9Qm!^CjLRSt-I9qYZe;fwI4+jmXzQkDG-QBGR0Fe_bF3rh@;5A>~z3eR&1hFy01C_@0WMr#zHp~ z0pbZi)22Bs)B=_@gRg3BcVzOUhj0^4KNNpWW8ayouqP8B)R-pLK2*u(5=9tpxmj0I zsz@dRqIEFIuv{CJITtoS(muB!uG>Gkx3Tl)b^_O2kzC2~%j8@fz@jyG{EMVPRpRyI zm z*;~Z*%e%qQo;23exlBV+D8Cy-l@s0*KX@6=BnPe7jHoM9H5$orr~B*8+5{T_w!AOJ?a`ZoL-zSAdr& zz4m6BynV29SFgv;?CikfA9)R}^DN3L@hEHSlA#FWi4{O0DLTg;`{<%W32~cohp>0z z;N%o&3!2-3=LH!bEtpY&2k2JseGj5&5td`c!f;Ft6XgQf;v+T)w;nFdowj6z76sJW z94sMG8V0_J=1LTuQ>m4r*WkD1$e9(H5r%*!Kc6mE5=XAIAmR$dpro6LVzI#smrO&b zwRmb6d?N27ib_cK9ZM9J40TZur!Z0|S~@VxpNGX$|L2lFd~k;dC}N2+T6Kse4*Pf;`1@bXsYTSQe@Y_b0Ll#2_6X zwt>(kDNnxJM1kJsA{+ja&MpKR*V zum}WEQ`SBoH?9EsvF6FY9A)^O8GQJwm?u@nz-@J5Igp%_DO)oihC5fTkj)_fJH`|iUT9oj ztOO1E?c&wOCz-%pQlyC;@;Z*#Je%5+V>r{lPkZ=68NFAW&fPboA34B&Yh!+}fLe>1 zQnbg$G|@>q|sgeEP0Y22WD0PFB^xV35fn~g-Nt+rLIky{>*d@6tb|@kA*T|(vo57 zoZxJ&3Y;VC4n(Mnm<&$m_|-g=r&a4b2){43PYuSEMOL;i3_|+)nVNQY+?)ekt6rA$ zd8mRx$djm`M5#pqSSMrtQS6R{dfMj&xeBwo31G0<`JwwTa2TbAN;tM$W>xrGD=2(M zEcQ#ifFyiJuAE0k3??k#at}n3YpO4c309fYgO106cfuxp{SEU)_>di;uhTkubg4Dc zw89KxkjLu&t`4k;qwy7~0iee>YU zeyTALqM818@;Fk%LXIekhS{A@91t5B%S4|eXwRC^7e0rdJFK4X!_Pap^Qh~87yXY$ zgjD(?>JRpnccav-^EtZg;ogSHn7ys7Yk((>u4n8}=m2#sCNRA^sk9$ul84k(h|rPT zc15iU5~G*Q<3Xeon*TYNoil}SnKX?k$vKm3U&)l$VG_eCOy%z7``Ic$5-b^OUPfA+ zbC*&wgLP!z74>vXR#NUIa6rN!AFf{ww}*+f*ve0&@Db-#Bn&Ysq@g|hH`upcOJApZ z#^v*JYFb)aYH~*-drxySu2-PYs%ZCC()Y~N@E98>udgW1e$i1CYxB-H5ddi%#Ls@GwW@Q#n@MTG! z&fo;DQJ&tALETeX=$>*aOX(Ipav39b+g+EGHwl`ND)9anwNvZgUCUCGoqa#!zMuu`?Uo#RR4iz4`y(7K6EWPj=}Qj?2G=H;t!U<{#JTT~Wh?8)C! z{LJ`mm=4NS<3t(5b_j#g{dnSYzT$SKR~EzJff1Wp>Q`371Tbc}V2iijaS|DP8ScL_ zT*h)-X?d)^HYKmR&q&Awa;&gN(h^4h^U&-gq*hTN3dRx(#&x6}PvHHvqDdM*&T`by z(+PDE;#OsGzmUr0Wz8qKOWz#gwjlPUFboZK0c%P1aU4$uQJ7-RAN}p9>kW%c#CBMa zUe=f17509;CGSM>(v=AfEwXn{D(TNIJ4;*RZx0&)Cv@AjGn5YZH8|V|XVDLC(7fwz zj%ktIpS|qQFc=YcxMA=Yo()9*+n3@&evxL!UpcX&Ts(@(r&(9%8N^Tj>8b&{=C{A^ zA|z=%#avjo-`%2nKAl%)v@H;C%q0fjo}n$?BOyxA)gv48Z;B8r^IE-Nvvr8RjRZUW zkoq7kyW5^R_eO4)zSYo(C}V+5tOs^!MV6QqG=IW+oIi-E^x9k}+ZMd>ImIf|T0YCa zA*>|VACfF#>Fx?^Eza)B_oNyx-^3C54IK6DS5eY`hrKcYX8Agq&T~e2+9$(W=2oka z4v{PytDrlXH(KbpeBj~TxLl;Wy{<(4t-4y}+}7EF)Zn0Zh5DboYv&5@bjC<`F(d83 z+6AuiFldOT%^&MCt!xMv!uF3(leJ4eFULI~(5LT<la2~=AmwGnv|CXF!fatc+_?E$6Z() z)}}^6&&%0*m9rG_nWn=dpCsMm?Wbv`m41CH1n_^NrM-gq(K0nViDCmzv6`@-NYeQT z^E-9>?P>gsFm97LL%+$VfTJXyaCLT)gn$u(O~?Pm@{1^vRi602n=KC_n%F(!&Qk89 zQ0iUA>2!2Xc#xwxb)I_CE&ql&A+!FVc58TKCG)1nGeUSW9D_JNJ)6!ue1Pl!YNtM@ zsPQ$)yYw-jxffJ4Mo6~aZlnPB^ddbZKd&6=;28VA)7UsQ?jQ12ZiuNg)bG`)x{k!(*UvvrAfjQ8za^Zv8Z_=D0A5;n;q>g(vtY@+<%*o53HB$^bPQwGn0mqpd!*5 zA&-Sof41iKutWOn6|aCf6fARJOs~qK;Yz`%baxRCMYn&BUoqkr*q4~=8-uyEUA|}^ z%5+9~k2844clC)J6&_Ep<=6Ch*}2s~EX5}#VhbgFZ!d?x#Etkc&1~fEF2`WA%|cu( zBqi)qTnvU?oXq*OwqRc_MJT&N58l~8>Fe$pEC>{_A4{d(snmY7m;X=3k%>KJ7t2|krQE-rJ%s0>eM4FT8>Yd;>?|>mcP4g z5r-rxLI@oxZ&F4LFWVjH^=PdQsL(CH(t&wHJQ5OL?lO!P!riLn?*z1JlGkW4|CwK= zg=%bX>)0cf>F71=dNTUQtItb%SKU_q8i*FJ^#5mr%%sx@tdPw4 z^mz-8r$MVqg4JscrNiKNwe3-#ESRE|RGyEvZ_c$-?s%{psrs?>`oOr)*2t286Bcn>VawwUxK03@pGicHCjMMKI@hwt>nNK z0$IgoqS>2u6+`UXdl+f>tBZbXO zWn?@{lNc^>g^)=*KUlxk(eMzaF_h(~r)0K*h%Ns}WaHGV@8E`jISmaNPZmE3fNr{S zBoPm7-a%*)xGrc>r`C`Fkau+!*YIAB65BETh#km`;NBmev19+?waWhgoT;1G z9XJf+eCTMjz6t}VpA(|hm1{5XQ93-mTw>NV8CaY7hX)vb8J!~6!eJ3%ZNBZB9GjHx zGnVKfj~Qv#VqLfdVB&=KqU$Tj_~*RfgvmD5Z>X$A{-Jy){R_aM*h%<45#tQdcPKdZ@;t~P&&Wtc@4L!_jlG&1l^8Kxu(aRg-I-yWiBO0y) zwFGy#FIOjyih{9Z`n{W7fgg?y^Q&ei_-~W=LFSRTOpU&J0ZWW=q1Iz1i>O z3=lK;edMA@?dJ!2J!cH{xlh<0`C+OqAf+wsgU!+CNXPUT%{6Uc;8{xC4$4VnLxB66 z5fVz{d~R8f1^n@;1niAd$YG9L0o7m0e_>RFW6XAB@K=&hHmB~NhU#-K9$=wfVWo!* zKA^Z34@%MDM*|hPL_d|G2@*$r-6;+;op@>>5kU<$qL^}ZVdvx4@Q4kOXqBIh3=j%I z#7O%U)|cu<5E?s|^`A!*>Hq&|`VJHhsija1NFceU^JvFz#xnAfCm|3v2&`9xzxM96 z!y%PoiS=DIvP?ZoS8z#QXb|2F5#|()IwQ#nSK(M%ZWQ!GD|t|{>CYEbosEhi+_q@& zB2d~uOe$P$X)__!(fdTiv{vET^Ce<|xEXus@n{L?*dPDqPOWf|zmLGCM!FV&E=bzQe|5s3cdO|_mn^O@pVTI2n}G2T{e zkt$-+FAM4ylq6fXmzseM!pXH!4pKqo^LnjPcy5>-d2ucBMoIJ8vYBmhJl>&E4Q>rx zU=6yG!{MoE3lBiuLzrH1>Bi_KC~Kcg0i?%b{!irk`Bt6Qn=Jpvh$=ev?U7a6tM41$ zinhhNGlbR{PeODk$qFj9Alkr$3Z?m`;8$^zVNMPH$67W0In^X~)s3!7!cwo~s5mJq z279EMY}mT7_%9^!Z8w&nt|DAB3hui`4$l z;Z3uSyzd1jmF;8aZf!F$Vy9!qMA7RIU$`roy%%_KUqr6oRTgNl)n8gsjE=wjYUxPtk?Vf z5IB-Sk8fsDF#M$wiCBX)LLy%plCXn#)zy`qI)gG?`WT)ov(A-zW%vv>ON36);@(o)rEK%+aUZ|4}fL zgFt8QjTqr_AqRuFuN($}LEJm{Gyx9QX%DH$h>DQ&^vaU41n+DwVQU8gm|q}|s5Zmt z-V2;0Zdr|)V!SBOanvRh}LL*4UM6RYO$@LkQ2Yf9#xLk45Ao_ zq{`w=ll0jebH=ycy1}+J26LA;i#ba8zjpA)gZynzReoqJEUbXBEm}R=BgsBi&@qLu z+@utQ^Iv&OVmz{%tA-twE3t*}{H$(Y0lW<`|CE0b2j2tJa3W2}VF~ih`4|~q@{w_H zA&Va>HlTJ>N`PAfc&=4NU;tmL(cXDTvn7sjn(!k^Bor<*?tzpqEh(#=oi~;Ss5~i- zzYIx747~V<4h$&c+&cpf3pM^7ny+$wNsU+`xl60vL1rF{0@0$SzmMU*>{BRQKW&bN zKy0}ZygC~K3G6fM`zm#^DLJ`STt<81C zMU8HDZnN>c2qc#78Vlir#WPqt(}>&0Wq@c32+JQ>(E|8@JvWkwe4L5Df7jNUgFv=Z zNyk+^&W;kX#Vo25o(GMd#Ij9zKXlBoql5w4duvXkQhcI=3L!basoFuR0vYi4`q%aA zeVzY$qq0+9|`NCUffC}k5HmRyvQdGn` zHt7~1(i)+?qTCKCYt1~pWy}b=Ic)rOn%qN5x zme1c}PMIR=$&b~|yXbMBI+2l)c!f+F$tU`9XmZGqlzM&llmAXgr^@w;k0? zX5OA&e)T5~=ZMQJ4Fzmuhe-{f1bUJO4fIF6q`x>h8R>$!hako4bL*%@YuE?UA-*58 zJ<0bzpaV-<0J@D`iINv!I=#;E&t#8>fdPiC-ui>XbPNwS4Ld!hqZ=mtIYJ9r`EoS0 z<01Ysw|R`R^t@Z|pli;bm-QRb!Y01OYV(5&XpeRNxMV&cW9M((GEFo>^uUsHo$_L< zNmfx~CnJzb=DO%fE6SbjAD%2NE-{V5T_w>c?{X!Y<#m05?K?6N_2O^E&*3QcysN^G z`*c|&c3=PCIiu<y7z(^u-KW5s8v>2)xhjkc~GJvd&qhDfFQb@=)fPLVxzhul{p{Hkd z5_KUq$W_=Gm~QVM(pK+gFi=p z#qVwiGcMp1lk_xAlM~_<89M$gH9V}swBC~g(+k@PiqrGvdsnNN*E@2;$JZD^tcC`6 zm-XWB*c-?wyY-}-hQuu*eY?eT&fm8gO1Bl*?vLBkx*|sy?HeaAe|&0f9IJp}H8PNA z@)hJ{I$2FM26SfFFQP0gilqk>CWhZywF#m_d)qz72Y;>hL_~nw2=&7c>qe_kzJh2h zZMc{28KX^Ez7?-9tq3Ln2==*S7nWpj4zUpAUXG?=vylOX%k$U7@{nTIUn0Z~(RBjA zq#$#EMIf-l?^7uJmf<>@ZZIZuyp(2AP2Zg+N4k z1xGsPu~vKKAKWuQr)<%P4+E zE|qzDL)ww2)T=RMd~nk2e-72sr&)=9n(g5vw5 ze41(*8f*+|59c ztRI-vE#~}h7BI%F`kVg~v%jrynvbJ7rx6P$6PAB6gaOLt=27`o^pA9UekTj@egxN! zP*W&zVkizAn(EosJ&d+0;V7l#BEAf}x`1Zl-RVPEu|*+XULSLHJ>!lgi_H zLy!N%V7}Df?tZ$2oK6dPv796zE5&Z98I@_(kq>Hv3nd*D8q`jL34Gq}0E|t|+aten z;+MVFelRIH2ZgxLs-F`TrnoStt2|_J6xm~~mUFT}dV1B~%kBMVwzT4}ZO&x(3*c6K z%dFhm?9ZN}i%HmCR)`E}pUAA9x}zo&d}ne2i%5xbCM<=V=Bxth%&VQ2e8q&sqsI0i zL-Y#+O6rLwxD@WuA<2ensZ;$8Lc-c*HM0BaO+r8~-k9}I0O)*Ei&TDl44tOjJ!ula zK0ieGE?=fW+W+nf^Ly|$eISy`#kxxMR3LZmU@`J@2}kP;L)g~_ZNmaPGKL%&*IzVY zU5c=%JM)v+N&z}NLj7=&J^?7%SZrw5#xMzHkR8NPc-92mqyyP*9BzLK)k|8;YH|3#t* zM3Sf^poxf1rh?2~3}>5)#gw1C4f!|yFpZ24gj#3sX8$}z{6hP<{Bl1 zMY*TU5sbtqYb%DaU~WB@;``{^P)b0P{{Fux! zIKo1x&pNxn19u4)$WBcDfD^mCQFP=dGJ;A$-)2AHSs%mB4ccp@|D;~LTHe6c4bdHk z{eB>2>}=_2sO43iA)}FH~#hR8?y4VuX50)0j;1pl6wQ7e)O0WrpFhuS2W>=NvkBL zOpE>MbS-eT!P~Zgc0y)rP;l%14<9xmSNRJB`Qr4aVJ`I0`{(oaqcPOdzW3`zP9+mp z*neXN*7&cSYc*I=T-1#7Cj{YGz>b<(m*FSdgZbox_9qPG!w_Rp&b(xEVMY#4Zc^~v z+X}35cyo9rausFaXKc6=NCZ0(a;)aTPH32=RGBa zgyK_gbvWF#{||o8LBqlt`bTX7Sozfs)^8!2HClGXyRh=Ua~K9v5mM1)>TMG-6EGBW zjQ4|(-$pX^BqES$!l(axl{2ztWT-jjbEz+D#a2hf!lHU`egkB!Gzt6rIF0AR8iq69 zv$B>y3Lf6i$XI*cvVoe55XeJt1Oc!Zw6yYxJ{ByrOW8hcygzc33MV}y-I>{GRb9i5_Ri@wkhlNs-c^IMnVy-0#rNEQi}l{#HzH$d zn}b(}x!D`lRn=xsfU$(J;uxKn&#k0u$L1h3d-A1ygwZY`r}c8jJ6}mDp)MZ}JB>9}1k>VltIj9>rqWBoMs_B&bcHx7KgsH|iH@R#Z9?}~sSI>m{r zxF8onKpofZxUsdpUB9MeLHE}w-n;h9`rpP+8bc zMqwSVF!`tRc76lMj3Y4=6+4Q)ySQvF0TgW@>rDpK4#4%kuC8@GH78f0*(M7emXqQR z`#|>wZvarPIch19>LeDuCOJ90UfyktFjndVSMSCSa;KoNX(t-Yq8-3es*bd(H_zXOeNM6L!CWCk2ao|Ju+dKKc#Ou+UQP)0dR})Qbc%PZYrr)%YwhLC#DF4_cW2!Nkh{0v(2F1J z*xke9Z9XlpD3{3P6q?;wc``9IRT@wnVY1i1C#7d}%X+#z2dsHGLdR_YwNPX*g(%HD z*`CPj00(<(5I+9_FqE})o0}RtIau^gdvbg7xfa=k5bOTr!vGi&^sH4f^2+&nSz-jF zg#Hu9i8s}D+QJjVq0*CSS{j8q+dv;N$iP@1N4(rHo~4P;QjW>Wx`b@4hhpGs^sm+~ zgu&ue#c{up3a_ea<_8fFI|q;3?JNzkTnWUaL7=ml!{gOkHYMDY>#l~E7AsHMAW410 z%{__0gdJdb4%WsGd9>s1yAifVHyr;SX@^3}^i0?0S!R zzdn?>5yT@ZsIFxcj_LQZti9+mhP8>7dI|A69_2)wLw}}V@#1T7JVFK7LBp3VPlSMs z2v;=@aJqxVEENNsoJi9|JKaenSA;3x<-IL|lU?Zg#LIs$xhZNhYNBs9Jtmk?c=cr6 z&o)cXkC~pMogu#g)wF1Za4Ds%FP z%>~%GSx~ec4SjUnrz~g->>U(XS-20`T-nMH;2c* z+>sipEG0G<^IpKs2ON?VOoMp0=;6ytGtrRn-w|=NqDo+sKSfPR604+yO2DR-FFoEL~539?>y= zkf{5*;uiSxchb#kbRESk+-5cQyC1)KF!!q8rjNMl)a)+b1o3R9S?9FcOzmfz`_0FR zafKdYpNP-oqX<0y>BXias{KNNF7jZNZA3tZ!R@K2qwg@d@{^CquU-?PI+x_D2`@+z9Xj z1nc!~IY`=5I=>rk5M#kOMfGs;);aGsJMp?FHBCmFA?vP(v*VS zFaVLD>T7KcK!|N8+W_%jt{Rp?SmfkohLc34ebv%JH6nidWUyLCjI-9pM7A9=9jyh! zTd!&bOov;6{-(o_rq;q74^O)G=@(Nm&L*Irz9JFMBJ6rUK3M#;^=^HLjh&Vf@V*cv zwNUsAbOE?PESH}3;;{G!#g}x9J3x2x6`d=HtUWj0nfKGqY({Amb?k{+SU-=NE-s^! zrseF<@`x`2u3aAYI6-?-`JXQ6VQ@528pJG^q&8neFne^j7hgv{TU~9}Ukj5ABgM+8 ztQ4Yxd=7@&mKdzWe=(_cRldK}hqcyxEWboes!PO@W!4}z;e@w-{C(;&{*K10yx+G7A`Nh`42VI%jZ@`)PdAS0LuEG5 zGa5j@JU%AG)zUZ<@Oy%eE57&m70S_7y0ci9oGz6b6Eg~wU15h7^6tD)QH2J{wO;~n ziRETyrk4=#Cj|nJQ94Zy`m<-$cRb@A@zx(<1;b4HRGF?I4sY*8FZvu%wi_EgUO|W- zAOI8%C<;{7SEA0D+M|m1GyyYo_+2=1sZ{1+F|mLb?i#hnLCMh93Om9H!)JDG?(i@F z8W`r$ESX1(TU7ox3(%dz1!^4i-K^d0qY7*u{nyvA79Wtxz1j6_WaQEYJsnj69~>S? z@GtphLMN+47YDk|LoMKz<1F8}2*4(6I@bOHx~Oh10UB*`Yeze7V>+|E%_9B12f3A% z_DqND+6&xuzPQ-9ri#?b>(A%`PU64pPstymtTdX?T3=pzF=~XS@pYN#SxpN=uh_srjscu5EA778RJ4u%J{rx=LE&xu zPdW*zq~OVsX{h!>Xf|K?=a)}p+{hg{LJ#xTzkkoN@upfXU zuRQVp$JJZ*Mfpcv--LjYA_yZXA|*K>or;2VcQcf9NX^jQ{ORuQZiepe?(Xiqj`O~+ z^U3vS-oSi*$KGqL{atTR5yI>K^R=1r()W}5S)6KR^4E0Up8^d((l56*i-tM12G(-8 zP8t#ec|u~-tL1xtuF>VxGI{AeYJ)#h5)wedYYp)6?Zw5vNi$Wb;66VA)rH*%^r*@# zfpS`lz`W(}VwI&t9wPId9pVbD5ZEgv=k~mIcAqXB)lyFL+u=X@2mDo^{cNHDY_VjN zfq{XZWZJay1r})tJGafvz}p^W7Jb9u?>oxiiSKOOVGkMk`9}by(z?ZCZkGByj`_hT zm-@J;rOnw#-)}qq53k++a$({_JE3cQ_`K4p%fSXO=)K-0p|=S@r}U1(=-fjdO;eub z$jq(ecW-MebzwKhS2jNN#Cya}i3gE4I3D`GeAELsM>FfQ1~gtG8VWQCZ?0T z8t=gG_JGwK%^-kIqx^DuCzT9eccG*GCM_WYX*#~Z2&ydR#B2Q2o}BD* zb4_a6`-!>bvMAEF__)e=Ih^fiVWFgIi=Z3REEr5zRLdW3^RO9zB4`^~LrNHz@OK6_ ziiY-3ywb_=N1bPdpNbI+jM`jIg63<3d9yp=b=H5Hf>?tQ5Ev7gKG8MF6w@?h} zC=#^3`v>cdh{^elIp%#`n&-$`Rhhg`iT@>BQ?;?E95h*-52mK+NxfA&hhoy{>!FK% zjKB^7>5GX#QG;#f$jJ-&Y%eCBmX4}YY3hps)ezmT`iQnio4NVEDEY|+Gf?0d`X69L zjcZ7>H#Y76==rF`&K4C#{I63XjNeb*TTQzZA)wiv{7^|=IqZjsgMb76h=SyscTy8m zjf4{A$()IYB!rhEZ82bem}lJ@gHqp6+XCHKiYZ)xGW7l< zo%uAX6`uy+gpE+TED8X%9pBDp&RHgHNk>iFavpX!7KkD3^8WK=s!i{RrU+#7ev(O1 zRyAz?$577tMgY38o@Fi27!NcBp1;Lc=R(P2YE}a3f6tnY?+gWv>MOWi_MED5-}J5* zbt!>-OUiD~$J#D4BQF5%g07q!Ur>fabVVl(v;NMGniZk6WVAnE4`sq1andY>xplNV z))=tAVW&~&-muIDoaC@+<6DyShwnH(~h{fIT|X@)`hI zO?+4umDU!r!*&nG*sD;8W1*y%kaM*o9>C1)0#=@p5lJ#``-;Lw0{@u9_lJ<9=36od zDDgV=zn-Nhm!jKmvXu6aoy>&W)GPBKuTP|WZl#%(M3X<`X%?r){#`sJ%`rJ(=U&=b zNQez_DJrO@;Xj%Y|T&tHC64*&e6P`VdU8TBJZVxlEi7}5^0 z>@91E=Uvop=^ASgca^bBTKcjVzg2I9l=UJx_CTl)V4vxYGOacyFJ6&fyq6<|hNy zY6!m|x2NN;EK(KvE9$fNJK=;oy9dS0MK+n$wQigo?`1?ZwdGtdo-|BoZ?5@%f`!Z0 z$0{)Nhr1u5W0W?lJFfZxd`7&wu&;+w_jO9|GoroLbt&r5*K;ie%UazvU)5%9%qtCL zyjTt1w3TP}2RiSWh3B_sUITfVm9c0=j6Aj*KYtryF{#q#*Mv6NDE$tty)TWz4w0Z>a5`0Qy(;o%qmAZQ_byKY1gLD?P}#xyiEQ+1@8T zk162lxH~2xA(T=-0n2#IJm2>Eo5?fLDI2{Iy8~rii|SBHKFBLoZTu5`hGwW<5uDQ8 zT=vM~pmU&#?!fMwH0JU-hKb3DknXx@J}SetUaHauP;$cC#G^0(O3n3H?J|*mQNCCy zcd4IBw+?ID<#sc(dZ3#WZ!Q=lgBH;y0r?NBUJ zzaO&k!4mLQrK0|SU$qRZ_Xdkp>)Z#;FNC?}zG&cphoQ^efrKaUQ$(5IYxp>YSF4AW#1+ z?C-0>U+kU#BEH_=y$o}%zM|cG?Y-lxhaHde8(SImwkv>8ggA+WX+WxJNoh4m!^}`z zvm}>|Aq~*P^AYoF2pQ2!)n$7q=jvlG@E3c$JblSysbm?sKF)JrD0fVBB;f%DwYQBV z%6NAtg7zxG1MOh*Pdzua)#P25lue*;Y^)c_SWas45n&~h5%r^EJD}K1457`(qIGBMzZ}aWS{g$wB*k5GA-awpjy4ckagh7)lY|Syxe^ zt*p+@&mBe;qtwKr?-ikjxd-WbUxnYD^D+E%^}Pc})6m9-3!o($ZY%ATR_0lFID0Es zmaqAj33MQU)fdB0mz`cYU*WX{IeCws)wfvxMzh}7ypIpX_ygJMRUUP}vNE4t??LU> z72oyNxXJzuGJ`$Fz2;;V)|TypKZaJgdeDTQmZys9K;P{W#u>=oQW!0c;tUT-qI36F zQ;9F*=)8uCp`yM$NK^-NHkb%5wjW#q(&=&mPELFe>|*UE>v87m@p0qp4D(C)K%qLd zKfu$~Be*=^B*)PR*MC0*Cl$2YZi@hRSms^hJRx+Vk8w(=Jg*#nf9Aq;8UD`OfqFN9 zC8h(jlzk)oRLkWuF23jSDa(Bm)~orlHJ-vzRKX6kNI{IewFueXI~^sBzCdi+p-F1M z1`J3uf%mD8buD%JSW+c(LuRWkL+YWd-lV}9K{?$GrJ{A`9W&F7WV!~SN_gR!h4-&j zx3jA;TdFNwjfO@>e|V3-=cGs*eJedNC2?|ODn5k@wzeLWT<@*$I_5Vtf>`MjxTXFY z8BHys+xTpF92ce@2qY#hr$ul6`m>N;0?&CX$+|9x`q)T7xHFdNbuNv7sjB;;I1tkC zBTO!2CS&^udT(NrN{ZeUzvJKfP!>VEU>!S>G%w(t?V0n(apw&iL(x&GhNmD zy0VGqM5x&AYa1h|a&ul7Y%+Y@n}|c+nSM!IRXSmwZlkTW@Ai^kpUuD)tly$My~!?t zVLph|^98J?k5iQnRDPSpO_ixH$mm+*l<8a>(Z4KO2|n|u9ljHUk2_plU;1y*Fw2@S z+b45{BQDvGYJ1}L4|K+x*z9LD&B|Suo2J)cR3*iCb+m50Uw0q2NQrkojx^pJ`}e)Z z;f9sFq4QK>HsDk2r2>fd+F|ul^`>i2cQ&~9i3UW_`^gw;dv#MYx`$014JdU(=jq4# zcs#yY#!SwuN%xzDc?$5dRBgA{xX;smcXV4@yEyQD<=%fRdA279XgP*T!pIx=($d=N zj*>mp@vg9!w^&FE6YHG(cPff$$XLVhvNU>bDk7tIdgqj)uF_qBd>T3cWbGw#bo8!{o9z1 z#YoyQB^d&y@0A=Dr5Vw>e~5Z*8?@;u$S*zQa>IrLq7v^!u>(TA2s54O#F)?t{8`yg4{M;Sn&Gf%o zz}9ivlQ6`tH?&MZgc;kc#~-cK?W|~{^4pNi8|2hb#+3YwJ%tQM#1(U=`;@dzJfOLE z^*pWfnO}+hI=*>bnRFQ9ti-6CX&j#%3B7pYr8VMgz4rllDK7YzV)>d`ovtp4?Jm=f zI?pyx+vHnAI|!EDpBQ#~t9Y4?9kXQN=De$g1O$#-`&NTqKiS8zs-N^-uE?6Rl3`fu z4CP%d(!J`B?PV{-Bf^{J9Qgbl3>ea2O+M6BMQx|UJT5xEme-WI$gPT~O-NpXhSwxo z5((l1U)`4V?mo?}RlJR@^!P8#8@SuWS#2GKUkLW?W{O;)O-@^xo`|`5KnmQ*veO%I zS3b~eh&q;K5U|a2*?qbM_)d3lYvO^T5uInc-IO33Z@~kpf1j73?P+g9`qqCn_ONe=kA3!c4j-;CM zbO!0={>Zzzq5b+$DYdpZx08KWcPO@_MU%z=UdU0+;WTnV#M4icx&qv>_3^oiTWIgJ+#ha$!{ zH2jT`YLC=WO-xNaKe%_TLFn8Xe&~VP{H1a-yK8>hSUK6{>p)t$QEg$7-3IEFza=4= zS5au#?GWM%W1>Xy|E4mWGfS0fuWV>If9Q5~;G(3FLAghL(7rTB=dkT)66{HxA|v?x zqQw|XGmJe|m)vkh_Tu<*A>(%A`09wcu_kVAs~?>x9XE&=W!$D`p6#wsYz{8uZHB{R zQj!k2J@1SGma(>MpCd}g-Q3#psQ}?AL*~_ zS{)1tS)0dyV%y<^YV)5vDCeHP#f=G{!Cf3Bck)!%+WHRT)X(R)bDdtk*S=@dpl#%_ zHEnJB)cO96uZ;X>vClXB6+qV2V;1J_^+*59qF>S8vBf#If5Kmm3rgWH1m*B5?{+Ve zXk*i(vSOpw$@Q*RgkStP6vVyLJO^D6~vF_T=%Wzr!2OG<S-v6CnB0RDNuMtzDUT`@N23$a|Nxw53kMRI?L%fvuV6&F+VXR71H= z*BTnc0H$q@G-{m>Jk$_y^Qq~-gzlTn%$Ob%W|-Orw%|5`LR1#t=o{UwZwB8Hy%|8E zM0>8o5Is$gaV9_-GW$tisXLYK1u-Ke+_~p*urpgWvp|&FwPG=!S}nP`m4cQG_l?Ag zmwpsG8pB`5H&vwogf*7zySo#|qW}-!Sr=eyucA}+j`SndbCD72t_R%~Q!W@co{N<$O(YM&&UEJ0t7 zqN=0iYW+wYGKJWmk}jv0v>P?|N<%|)zCthNc7D|oRKCU+?5(UQZ;Ba-8ywM$ zJO(C8PC0j_G8Cv9#m<*7-h#!J>`!$rW?i<_~GIDU|yJH z^y$kEGsUBZ(UDK^e+7UT z1#`26#LeCGB0&7%3eln3oaFpQ5%Q7N3#5ClIn9Jqy#iopE)B$)eXToyZqU+sC1Um~ zzWNtZviDVQYs><)vK)B^hbjV8Y!%qN=kd$PAT#m#cxd8$DeX@R%F@~rz2~(flO|`* z2f^ib$XsPodK|y!6VU2fyZ>Pv$(H=giE$OYZ`85S+Y;8~r!oDKNsU z>`nHTnwYi9mbJ1nCJ)@V64`5j<1WC-8+c?ik1s<+xN9u8z@Jp$qlm$5`Bul`j4v>SzZcy`0gU0p51&1w$*vDDj11?I`-Z41rNO208`1lP40>iF({# z8t-TRKGxHnqr{*@=*>7+EOP`~4X$9!(zuOMGS)aJ@MIpLlv2uC*kPb!(F9wxx0|D% zQtrV1!}y(JSUG!;dgRBoy`I^rab5li@p5vAk&cP5Qe4P+z-RtHLZDc7_jIC*sF^YG zneQ~2sqb%1>uC)UMA4lC7iNk4z(Ge#EOVMAe`zTd63)`9f)OWacbB>1=bry0LTn}@ z@0{d@F7iwJ_$01LCYtvBHrzWkFx$mX@1Rp2&P9;d(|*=(p|jjEFd7Gt+ujS#NSo2l zkcAOof8i%j=cuhjLB|VikH|#~_Ij1897a;W^FB#2-~%0^j4`NucarYzQXUbTw?HPI zvf6F$xVcrVAF?**ix;|!>@TmL3UsxoRlc%(m$!4Pn18Jl^Z$HC9oxrToDCUE%_k>W zc5oKf6R*hedCFc!_lrcU3SbH=187`Upw`pK-{A)Ih| z0ikKu74Mnd2^v7|HjHU-W{B=b+87`WSH8GFw<)00kzs~{cS(19ZFJE{`D8HS#4eUN z)vi{EV3H?C~Z|RKmy;(oxY!E_SO$|w#`SOacaYKqA6|`KFdyp5v!_erdJvn z!;I|>xdXGE>S#bxLT>QKN%yXgq@{}fEB`j{{+!M)csJLx_#du6_{|g^@MtPXg9f2LS$4oF7+ll7CMD^RKHJJf?G9dtn4P(ro#!oWp^#cYDcgoZ_ zX(A-xslWEkqxP~wu$!H1Pv`a}PG2`#rDQ@REA{r(G_+9k!c-|)ilbs<3%JCFWIg@a z*s6eh09ZQ&DX^Usll_A5HY4Fq zHT?}!*R)6_+)=>hsENJoLn7*dfjtYHPE56-ctP_;In6!Qi;gN7>mnY*m1NHgmaA zLGyPX*zDy#%f*A0JZ8Uh)RCX?p@=x&oNgy;TUQJiHzmKF*JGw@NgeJ5xlR<=Srzr#{S_4n56AZ4+ifq? zQPULSE-10;c!vw|0(o71c+&X7k&RWyf(EdoQLFSQ#~zvfQwaQC74`P8(&8N2J^dcr7k2*!(b9*Wi>jTl)3AD05kP zfZ8UBQN+-sVst@C`Jbs+7kl@ct_JQbGHxyv#f6~zp8}_SVGq*z9xL0&VEMl}1>*q? z1>AP~{L{LYrYx0$-nh=`dnopBm#GCe67KD7O5d=o{1D9BBnMREt^0;g!V%{7k?aED zZ&gLIbq(@$`ibRF=4eaWT|n5&`xvK%A{WRO@f$yDZ8Hoh3*fvRd8lFRD1pA2@vF+~ zjpZ3TNLz$N1kK@%e#4(v%zalL`^G+4kw_y#SLQPqPMCK;OP;0seIk*&#P4LH9Lj@;W{=;_jY%ujFJ7yc%waLh%~&PK1>wT)>*bUSqaNRGGwpqMw228 zx7j3hE-xQD3^A;vf}91eJR~7=J*ofM61l{`WJJ;iM$!sycwsC<@|dj5i-9l0=;)}q zWhvoc-C0(!$1BjrowZg;F$+TF{9i3VttIH|0m_IM?wf2-gL1FFM~Q=SLol;L^TKVk zM)PlOyLC7+T9OSG3xEF#ZgoSAw$>y<#mAc#H-f#@jn}Qz_t&gK{rv@sR{P+Q%R+l* z_M2=_!bkFMul=YKm2g~RBSY7_;8!{I5o}?P_0GUNyVdcx%DBf-YLAJAW@^L)MtOoS z$FH|Su6DntJ$rU2L8N3yr!AMughy}1)A?^LD{oz=r`Tik=2~84cLOnITVH?<%y>#; zq4vTHq5+3iqPosKVyDF?IBL*!_&)uknUJQ*)>nqI$vRY)Qaw09CaZsDWe!nY#;D!c^1B9Za!U1cVb0BIa3g=HcZn1G1r(Mjx_J;uZc1LeXE0&fg>|+ z5FZK)|6d}wNUvW1#l^;?f89{(MIYC7nDKd?UjDl3$J5V960(}AVp-$rO{h;aiqSrJ z5*^R@q8~REavAhCfa{c3JBwus&0TWy6BAQjj;U3<)1BFsNBCVQyStPcD@66P%$5_k zifPwd)nqCvx^5D&5Gmb%*LU+0Ko)%DaJuOY*e_q?LnkM&Z)p%8O1nmD5wdG$o{eDi zkj~7^uv8ycgbfXkxV-eb7nE8E*>lfLtxQ4jo9@?c>db|%-hH$npHs2R>#M#KYf~BmN~B^-w029jYjLLw}TR;Ir32hz59Yp!lYSJc4UH(Y(->pu2A>hkcanqh2X58;mXl9EhR%wZJv zNS9IeSvrYA-68sl< zdi_?uQsr&!5b9z?!GHY}F{0ivS5}U*Y3%X?!Y1L^H*)>wK>k7@3c^gbx{g%SQt`5J zZZX8kS!3U_@9fJy83j4|9pQ)Itb>P(Tprp13I0mtq9|i0FFzWY{3RDowE}jfws(Mc z^p0v52#k5i&CVCt46UXYDl1MS_T`5*=b8SpMV}o*5mwf<9WLqIr!0co!?M-JJ&5d_#6SAgz0tgcMhjXgl?2)Vmk zbDcWtAu9?x`>bO2Sb&T?l6mskj0Mj_UqMx~unf&Esnlvi@ffY?d2xuvuZhz91Lxhw zCynYHx+$3P?ED~*gdrT3mSSQUG9MYOl=`-3={&j(zw}xYuVVSzSn}O9#Z?qtev);r zu6GMrp`mW4LM3!1Hg5aFjRPF!Y?E5}h!OlXv6S^9T-`Nefvzo#JLuqY{TQlZf>tebX`xueDN>kc);KN#B3{Awk)SNN}jw2Nc{OtEZ?Kgg+p+<0P^eP(T~tuUJ(hXQODdru{0m#dgK|7O`aviLRC+E;=RtU{o&IVl4R%z$&_nTc?)_yc7CqDy`GGosvg3VI0AwB@nb247%^WgB;n359*6R1e;{0;!UpH^ylF-T(ae^4}_nT36d z;93>lf@;XylUMjfLoD^|dwvlr6 zo5*}%)b+}VvFq|E%ijgJE6maZe^h9XC~Ed+lfq38hp;iT#gUDt>e~!lczyrSzz?yW zp@g8#;*t`;)3O3Hg{Ywq5kakT^&@=e_up-;T>_1Hq=0#ECx)HQ?lm|hbXs(xB9s5A zbgNy@>UF>8VIe7%&ejjDu(`T(S?6|Eh!V=i&hb=i| zE8QR6SlvNWW({L;%|DT>ocLc7tbVG%DCT)*8l?WK1rNiP(R)LONx!}mxIcdFDd)0! zQyaU@P6u-4eIGj+LXr68i|V$5iDwcvIV0`2`<<#)XE3zpw7{i!%tWS9^*?$>N{GFELfNPSAIv0GRt{36-=u58 z*;hWX5mN_;E%9=@KZAw}!1I|PGW z{PK{u^Kz}-KZr?>c2&XX`2}MxqJ^?m)&Jb4jD}T4?ceOGN!kW5^E;tnmHo+oGZp^o zpyyDSU*9ExD>pv&_TYFuVPEW&R83Y<)9s*x7kI^89_LS!Z*}66c9`3`7<>{xkKe&` zrZYz#nVs?=Xph=e{Q(b)-aLEzitYL5i^)%3x~A39o`h0`l{bhUq+1NKf&ak)5u`Hj;o zGc=Fh))s?frq}2CvL&T7+Y3ipOs!k!6sfL4>}x?6%#l8hmLG~parMGO&lnl2_qRR! zS~)dj3?EQ8f~+x>;CcHPEB#-8&Z~Tx*jnm&`KCQ>INGO{hvNDj!GgRLSeL+a``-zgyP4gQ z7`DUvPycpnNWaj8Mh1>!1{NDYlwo)hj`zma7 znf!If9onjI(PbzR8IScl$p-q{u4D?=9t|bNP)!nNa{hd| zSF8!l&DHwI}d{UMFKZof~e1rHKusQ@@Ko|=^-I5YI@w2Cdg@vV@f^@IByM`RW zA6jDiwIh-hmF4WNwti1)1wen44~45GDX@1$&Vy2OcmacM^JVNQ|8V8Q<;NbHzQ#6dyMnXa()59dV3JnJNgD!+Pmv*jH zjPlK3Ma>80!^~!vkM#(Y0U71{=dI8OT|2w1%{lIAKVM&GNv$oy*gkkrwr2BHVDC|W z;5)=aAM~-fecr%mvzhT6D>_n+D^j5vCLY@6?N_cu@Vy^5`INar_znZ z?bUtr)J*D!{s&4{tY$<)5v<+7hYmAq9BRB4O&52pycmElzKBpm-lS8JunD!b5T@i9 zG!@j~Wm!EDMFS|z)qXtB7c#_mUBV0;LnFFF)qH6v`oGK$iPg_XLHAU)zq?xPSFbn5 z=d54Xv-0F38<^Itz}GP`DS0z|APS2^zQEX_hNVrNTA)gHhsd)cq}#Wl5IIzAB}}~q zy|BQFXA0`m>rJE)md?NK1c7Wyjnc13wT#3$4sP5aLU@KP;t^=KNz~LdL1agri?)j$ zXiRKuNjSNS_JR3jBK-5EceFnx4o9u6SXbC-BEG?`T1D1PdCf@rI z*D7NBaYFCtDVRKUVxpDU$pvoCPOc5zgxXcFAY4uP*$l+@*!of$n)05xpHUt=Y>OuQ zWF@PM0+7}F6mRs1N%M*!{V#}-0rvv_}RQV31#+ljVvTQUt0QFx4Y(= z9)L4MFc>GfnLOC))PCv>iXu=Qz4`p8&#&&`vjvz%q$VcH<57jkSRhvd=DDli6!U|y zIWQ5ya2^kSdqYFmhm)XY1EVT6r-;^=@uIT)NEqdx&k7q_-ZC=MVkN}|D%Va?v0((}f_^~Y6wB69y% z3m|3G&!a!1=E5oL|46wy>>sXNUZ-ABg&SX8RBCax^Vxx)`p*G28Tw5y5j>{VD%_SyhkT1&t_}tp5eTk@GO;eSX4&r z^T?W%`4zUT!#|@*Vz^&+wBr5j7BZH<`SdM>;o4ta$yM2sYyk`3sZ?$6LpJMf;h*_x zd70p8{(HLEkYdT4ZtCvW?ObXa8kxU53qCVpn@$>T!x5Xt&&mb19>&fXe?;00q-`yoWdlQm||j;isqQKy_eem`obF| zM9vRzY6@Dt>Y@S*#O&cmpd{WgGW~QCu#o`QW^`!x$yvbyI?8oglU<$_@+$+ze?0ml4D?~<<7$}0D zE&?ZknNJI51ARjT^GMYaT*OSwu+>n=tbxH!c|#KV;edwx>Qt_a?NrLdMIrZ$;$r93 z2nuUmb}qP^SD=IHUPu^kaNdb2QA{BLOYC+fgvOMm^Q(%5qD{MQ)j7-OrF3UY@3r3? zcEs6Xvh4EE{6CUGivh4b|caPDhpus^77h_<%Xgy{zT2EWi3v&IB~x9%JT~gzeLwZ z9MjSSUM-IZ-wZxH9Be+W>aC#6B2WrjHHVIhiVu7PRX&9lMe?? zt&MjxqOSbREUmU{x)=h_GuJY6{m@P<&fDxhqMO!~-cjHob35IMi-YG2Bw&@zzL6B3yaNFMC7@<3`m7j)*L^AKz>VP_KxZ%DRV=pbqBU z(!R=et!EiO!nDJ}FwLzj%biX(G;P$BBp%QEE7s@t`>$)J^KdN5xNXR?{$w3#l!*Z% z2M{)$ZM{R@oyuxj)vkXS_pNh#@N*4_*VjySIQ|_?QDGtibBFb*v%W!j z!!yXV4Vdfh)R{KNC4%ih_dL*pj=8h8y+4A8l=28h*&eGNZ)mv+OpM>4bHviSP*C&? z=u~TpA{so}2g&wbR^E=d;)i&ysAOe%uk;n|p|*aw6V+td?Va~wNAab)!&AHwi*CHS znhA>nRa^@&oF3vLqV1N{tP~mJ4A7g7YCfa`>urv&Go}-O*SdA!Px`nhQBzIX;(nM^ z1Qc;k-8;5CYzfMeX$wNg-}AU%(*RL8JxPkz{OX1hVDiw1NwnJaWYbG%4oPEOHQ%(D zHl6RSutz7@sR)ql!9tOAw4K*<6&8bg9q$@(J5`Tztht@>Al~^gjHEQRYMhVG$zH9& zB*+0=|2DIP56$R-LX&GLe;vM-U~tM(vscd!cC^N6ksR4-R3wn-K%3vyYY$-BXJr>y zn41^}7S*t?BB)>weK#f8U^1cmJ^? zv9n`TUNuJ-qBtTc6&<4z>zv2vOFCmKoy<l3DDz zJ(gX>r}N)3sOL?m&=;=3rolsL<;2790a$8vO6Mujxrbf%|7I=wsFjprZ$0 zf{4saOglB{3dKH)BPA+q6o1sx(6Th;jf{w3XX6j^_M7>bzx*&A~ig{yfxC_u#$a|5VSE|YZ6wrOnt()Jn zC~shEN&?ZT+kJJ;Gbrx+P&F_e<~`4S8p;a6!l_)k&rlYCY7(WbwA$=kJnn{!XYu|N z`pmG>E3Ldu&g<&@uUwFgx9DX|FZoJ`A3n;CVS!I=7@pENH~VbsYL61N{a3pqZ~%r* z|8HwbNQcmrAn3o6@vUpSn;I5#C7oFegAWJY88g_`-=4%Jv-N6m?2`M~|Ki)BY#rsNhtWJNNf7Sbhm!R627Szibr+bzT~Pu{cE0 zR9Ndv&k9Jutd8Yud6{F$mk-Ab;x!qu zOpdvLO5{B6RxtNgqf}|ZU)H)Zzk3*NEq7%oiUsdAbwuRvt0cJ<9Q)%%$&+JqXZ#Ft z@e{~HLcnE3Q)FxJO7|b?nvem=Tm?}GMs`7Mbcg8M}zTRZ*8`^kF zUteU!6r^(lp^LXq(H&&cMSCOT2s1OILGOGN4wr5I`yYo59QW%dmD~p^Ik&Fu9ZvEz z_!VL<=i5iOwy997*QXPim+|Jc`ArRetI1nb9)G-14GPuJ+?o}|G{0NlOtWy zOB2!*+WR^jbzTEqcbDkvIqr5YSGTqyU$cqgRkx0O8beZ#_={d;HkCih1(a|DrU8^k~eJ15)yHqC2oO9ep)H8azQ z*6T%h{hFZrJ*Tbh*HS>3a*!?!J3?r7f)ZXT z(Tr$c3&45(8<=i_zKtu?sG}b6(qwE9+OiCJuGf*1pPZh;lsYf;rqf=}b#`|1&ar`u z6fUV!xcP zzOKIfK|D`UTgOpsAZQn5R6+Q0q>4}gzG7^t@yV}q3Y^Yu5FRjT^|aX%MYA3cOXvee zXlB8e+4~0j2C}hPT_^(K3TV77gwFWdn|C#HO7HH%dY6vczDi-$Zawd@xbTF^3fj=T`}{4fIMLE<1*yQ3>A$*Bs} zp-zoCgppey1Gl`gWNs?BFys88iCd8O*xt_j*3898Xr<($Nbq&B;|cYEaAkLoE|#nZ z0k0=fPPASG0Y)sNCoYbzoS67!gI(K7M*tp|8W9No5wclUQXEL> zsyesCa%}M92Y7`q#dT^OnPuSgW%Lyf76i^+oE7Y>{%6)}GjhBBa$+2Q;&rtAoTNj} zZBv@DZ+*F59kbnyR@d^^^iWbtij6(m9zaR59})W$223ikrTL3Kv2XNoBw{V@s#$++1h=gD}+nAsp?5_VT@yCX=(XzGJ3IuVdvx? zGMmmg7vv^bcL(A~(k8hFvw@?a7tvN2oLNMKr)J2Lb;|MhEKuC@4d=q^hmG zF{_(;wC|k+NFuw?Nf9@+2~yd-GW$FZH_I~T43AY}vt$|;tl`JoNXuv@8ZupKyrD~4 z;+3YQeBD~NaK>tS+~^qUSYj~J5S`PL{RHcts?5t~N5dxN>KoD2-?+cI9^GEVEr7jT zm(yU{$KS8$kq6~1_=CHE>l0DGXv%Zog(u~Gx!8aH&OuxYQAe3Z-+cynx5Z#cfuoh1 zXBdAjaYMR5U7+~lgh*#S-2)>QT3 zQ!XP6`9D;>WmuGb)b@*#(#dKfONyY z?&p1>9)k)`BFEK9 zl9-pp@vf(z%fsM4;u`9^LPOlPqGGGQiR;iuKLoarf!$Uu02!*mO))gl)SP>}?f=-V zm6EDqA!T{ZGbnw$o`J0WtTz~DZt;!-f1FjOFApcqYFN{#G;RO`$NUrip5#~tUsS8R zr4|oL>YZniFE2W)Yn72Vm!KdjiL2o9vVaGjkj**{fX{K^kEz3fR%yLJ4ae$#)sG2_ z;OfhGA#0bH+582W;2g=q9k2yjU_9mlV>FOlP*9&Bb9sx|Gc}74qFqN^m>}hkoSXvnbbAf3P?i}bG?cn{C!mspj; zC=03#_>aY##LE^B03`$%L0tSjeMzlR{rs8V?ug0o@|DM^+YkQN*J_ND@E>1M5#GLG znLIqK*SSPq_|g1afac`nG$I3OSH(x?mI5tOkc)_}_U~Xl=rFl70x5xe(&do75Fp+J z*Oo1Zeb&9S?P+`c`DMqsO-d}QS=z|JSFVSVf@^p3><4A5SKqo#IaPhr%iQ z*%7`z`G~x*fx>o&$Tfx7tP9dfjSQ!Ga07!#%9Q{aOd7d`+F>mx3SHKV>|>(M@Gv74 zV0iTC{5mF#OQa!!xc%h9dObn27!Xeg9y%h>d>+HeW&l^|V=l&|5l_TLuw0AKzHcB+Kmbj-5iNpIVMk^(Ni3p*hBfAuCb~Xm3T`)KB-zGL{MIW%hc9 zbf%_3-sMlv-&UV3Vm*_b(6jq=HIq;6)f#b5e&Akh*4CoGt6=UFbHffkV5AFZ_H2Hq z^T%;RqPf{ek3uRjQSA>lkN^rApIVcFb-b_3%G`-7gQTinPhIa^j$sN9z%C4K8GiTm z+gs81{p!*FP;sPW^zG(8)qjTy{Kb8(@{EqnacCzxF(<%nx1QW&@jZ0L__VmcPS^9a z_q^;qr$t#A<*2!`xB|xr$_S3gG}zh;)ND<6?^?Nw3vHEZ_mP+(!o@`Z|I@CDG8Yuz zFZ?|o-41)=@O{V&%PX3qO?RJxty$x*6=mzks!m#J>EsF?xl?v#FPUZj(*a7<$$Ja1CTeX;t2r;Wu`Hxm zGBQ^6!)3IzPDajO3t-N1Uxmfx$jU1De!T&A3m+oKV+eD%V%zh7gYUGpK(qtpLmDqH z5wFHs!r}n&V3*(XpAJkYr{Dy4(Mgg>_^>vg?F-uTx3EsK&nF*UPa$tY>3U#Dh+W(X zLFd$vcqXeN(4R}wykAl(1y5#kJP4#%sb7LZ@Jp}v(SkP}1~(K7v*%46-ON_)@+&H` z)FSKHyFSplJ|S9yq?w~IncFD^*s1H4RS?2`>vJHn2u=4&XB=qOh`^Ckvr0j~} z0kg91r z#$BH@Rd@KLKFN=K2u1PktKKZ=*=?w-Z>}#nSu9*OP5oZE4?4z|N@?pqq>}DS5pa0( z@K=Qs!%cQW`Bka#9~FJw=YJFKtnx~lH0RIP+*}-cvkixWG%8d@Yj*`Pf4VuaF4j>g zqIFK})p(*t4EdhHZgz|0Ln!7<)juD+i|)xrzXp;kQyCFiLyC`*WV(aq!kl&b? zD2qDfY9{}zn=vE^i1OyCY$eM;-)4mPA%}?qjeNkZVv-PnILcwztbtm~9^Y3X)?#TT z80cqn`df05WbV_;hcCJQ@1XUFo19%H%E3gDw*Igl0XPxE!=ONG=XOg#fBla~0)1h< z@tu93r;xYB&sC&b>>JCFbeh6#UyHZjR!U zuzk5AdM#r}VPjk{E#p@ff+$>-vl}zP#E%p%MLP@;tg+ojWbH=f(?X=m2{b}b%IjNIHkP=}^$qw{#Q7x;YjW^UQ`kNTG>;~DXHXj5rx?VeaUbNgh` zkaDlJ*w?4YjPUI|`7n+)pJ$JuMVmHTwfxwXbJnNTC71ff%EM(gKa!@1_I*ocL|nqP z(m?pU#R63qZ^)1Y&=PxZ^nABYO{M(isHX6Aka7)l?#1yu5NYneQGz^BoVuP&uW>Dz zd95x~z)M}QcYdV5IDXsS`jTyySvIr&JS?)(em6IJizWd=s3GU~>T2w`2d;#u8sWfJ z35`ta4}>vv#WdYly*2HI!j$mirBUA%w?ADb>KxF+ZYOeiACC~lZO6G6XD)jBn}ffj zK<>ul9SGZR=4opnv0ciKmBAg4k*#insvL>5T~E%Sef7zM`T5nopFkTG1y4rV{#!F% zU(ALyvy^_4LKWkFJ?&fSpV1l=|)6iF{vazNil?L~bGH;Q!Qo1$-k$il2~SQ|GfQD}k4nV|RC?r#_=DMzv81DLLud zD#4D!kMK(rkxz9M)Y|VMT^F-3pRJzfKX!i#@g;wSeM|3Qi)6H|xez-4>qySTy+rps z)V6I1Z@w@)(SXcEtc0ZW?3Vi0jqK4nFhz94Eit+4i{%&c!nFZ~ZBV+i zEXEXJ{wg*+CM{s`WC5Oid>nJdy^j8;7>_*6Hn_y(s=ZiNR`xT0Tg52TTD49~$|88b z*$-Ftp^qhcmX2WPXM~BMY_Ek$-=SIEuMG9}x?z(GNhvAfXF%`uc^;KkW!B2v(&Xvz zJVB2GN;6M+2eS981X-ZcT6wupCheSF=g`HpJUhjlSgd%2nYD3n3i8tPI@!6I`FU7| zok(=*X~>DgYs>KPj>Nb19Eil*Rvjn&62FozF#~}`#L z?8apAv(H`=J@M%-Uyq^;(hRe%>)BOIhOK8`4?t&dm4}7$+T-h5xu~P#=B+{ACQUpE z4mfp_?Fs*W^YDz(-D<65JQ>qD_c>4mH+m_nVg1kQ`0)R&4)ssFLwY;ta#d7cn=S*2 zYKGaS({!Fw5Hq>YnrQo~o{l;Xu--I(9Etf>P2H`E7CWD)Q(N)W#hrOZ64IuAI)q1s zD^<8I+}f>hzdoodZ|JynEBUrk=V@!S61Q@J-890?c&Iks`8rh2DDNk$h*JDy{dX^Tw> z*5jH$hbO%7na)}@mdom{Q{scuX)fwhOWfMBFXKD$4eTMgGVG1tsi#q&B%Iv%Dm;X$ z^OezWi$^^8zv({PP05;W81G|sP|$Mh+j+&hRaP^NC!c}gc@m7@po%FX~h29!NA|+K2x9PO%_Fy)9?PAA?VE# zOy^z}fmrJ~QFFU!>5BTTkZxum)df#ln#{{sdkCUBDhlegfqA7uA7qM6IQU%v*v*I&mMg}b$?EV`6m0XCV^4RExXK^&ri~t zCkrw`MpbCPg=V!BADys0{5ytmUZfqqv?+HhdpMX(u+?@!$A2yzQ^Z4!$()2JOh$P9 zJ)IV_FedAmlXOgezE}uO`7gvU2u~J%(+G0ggR;rEOWAu9ozp+jctIvdgrT?(C7-eC zOik11G32xB*(J)g)8!E>48m|Rnz?7I>7_ac@mQ)12qM({v8?!$1x}#2kv5oD&2mVG z5f|VlC&F(FG8l-)M%$$?`L#APTUBAT_XnWsvj6i_2R|cpgx_lGxvt<7{SdV)iDDgA z<)Z#&iQLGRK%59VM1(cIOL1_4k&j9ZIL?H}m^Br92#F#rvx4Es%WjcwYY`3;uQv(6 zFd>}4o{co+PLBb3vF#tEB4grz43Qu%X{gklgzOz8LGN6ckg$xYXobffXzRd;yd}h1 z@2T6d=l}g+XFg7gXrg$}g93fl0R`${OH@{Bh6t2Sy47;0f-@?PyWJ?>dF?!<;kcP& z#_Qz!y{$D{MQTO~m?y0DOqB3Sx)c2qhwMhnP&d4!qL3s7BAyG8MIz7J(C2nzt}Kp7 z7K-lki^7K@+_p;wz;p(2!%b$AXw=mc2>u%#sNmCU00RX40af7Rxn8mw6n<&>tvdTl z-}}~d+}Lfqkc;`LnkWg2o69(Hr=7!QZlrc*Rmvk!2k4=N4UtejGba0alhCi#w2?K~ zsNa)vpx>6r+|2O$!*3s|`|un04Qo>})zsZjS`~6!i;wwoty_c6lmqTOk`3$q?SE2F zM^U6G--x&$4``(sk%W4(Wv|yD1vjj$^Qk(?Iekp|cB+9X%`8P@egf;x=L(})yYDnZ z)$`Eo79^jq(K*~^%Kw13Q6v7TkH(71CT%D4s@-(hFIC5Wv^oaz;@)$@58rwAynQ%L z3%JVNC8u{G1TP=^=izg3m1LUAa7>nbC(4ac1Z=M}tvan0^n9n3m)cO91Ib4}y661J zQN%aJkMt+k$IJhQybIAX>O~&0z0u->`ASevtFrN=ES+eJ#5S|%{o4EfXC5QLoL zS^{6VXav}P(Rk{f8Co;n_boHt7Xy0C#7OoUr`)6kN-8>2cZuEYPD*4-I#w=7u*s@! zkhcgeW%GI?S!`!+1|hS;cX0H=ma&;+1R}wY`tmv_&)SKSQr~a5A2UsFPryN7_Fh}W zpiyb>7!K@f+)J5XbFTWy`&pPt$N z4XCJZ+U;a^u#PH5P*q!KvD%d*dOZ6Z<98!TQvF;CmsQ=N@cVTqOk*7~#3*lseOcs0 zeCIhREBmUrFCpgC(O==Yq01Ga8{V9{m=Gh@XWfL2+)RbxMeX5W(BTu<+*YgfIsgZ? zi7#^g!CGS3bo0AGLB7s3F4;kC`gF=CgrH98x! zN}SABEam%vt$Q$!l(Ih$L)Ev@mb^j4daRTAgI(ZS$PdkOTTHf)TZjbgm6Ui1nhL(8 z#;HAKxs;a$&G!f#GQuxLNY^-sSkOL0S-GCm8honjyOw^j-ddEmo=*h5E(02-9572x ztO1Rd^?p~D4S}DgH58a5O%doA_Uht6WAAW`%UR+gV#ttdeX?nrpc_ms&KY1t%a1C} zE3>a{O?%mfUFDt2Lau^reguZd(#R#Vs_XSdyqkMX_QdJ%_IuT-Zoc9!w(+W^?Uu7F zPd(${-(UX~-Kw-M|7Rpf4t(sgX@*A@ls7P%7Ry}*D%Bi`^7GKmD;47mtwjFQL+hUJ zn7GOOV-$GZr$~M#HT4UucWUabp_940Ugo|B?TY*%gX1jBycBfBat9`wE$92P-`=s( zf5s`2Qn01sSXoT6Q$F2id=)RM0FP;7bAh~#-PgH^ZwgF30nJL(r&E1!W`3nL$gIn_+0x|3w#fCr-fC)bjT{gGhz5Q z?C^pD`E?BJrF0n>-23C+$AFh}J`Vq?j8lWd*B;9~5ZlNDL(tmKO*f)n${xc!%c}Nf zXxEFd?k_{sD(!FT#ymBoW$qjv-AX%(ACg2H*G}Z$kYcw4hk0|TCW~G&ppY90nwcb1 z4Y}QXpN_JE%@3vp1wPl>YcgkaQ4O5P!@L82(Xy`$p3MAfcOsl6M0}vKSaWSjdA(J8 zteceWA<8I?FhrjU!M1C}|C}%W%X*2GF|GL$XoJv1YX9G3l~(n+fzY(Og6mdTy$)3c z&J+)73>tzkw5)`^S2;F~aA$_Kn*nZ*nOo1rgH_%&z@{m{&R2U%-R^@z_s#dUFnUJ0 zdu-pvP>ECNvVD#s#Dg#pW?*qzQ84)4rXN*aWDdBV?^F8kV$FsOIT7q6i>`wMb2IEy z-#ej;N!Y?0Frd1obB9xX37%b@eqyqF{#d4p?nV*>WkFCSN4uO+9@M%5-M&|q-)jlf z2`VVl_f>mftE4@+D_H2wrVkJuIi<4Uu{2VnYR!a*9yGF+m-oCq{hsixRF4gxUNJ#}H$e@Q(f2UO;~!Ah{(Uz>tnXOi*+Tit{^C*gL~GZP!__sizseqtgXBQCIM7TdS9axk}YI;irFDdleB z--lH<20nHjx4PN+Rt5(0OXA0*G@X0P+1+%Bb8`)1G4L~A$ZF5VzmZulLXmmQ28P?( zO{S5PKyFMLOaiJD&Tt2Z$F#Jx?<=18Tk=Cg#^}FPqI7lNglgF`-;wcUQbQmUWw`8+ z;O!rqo?zdhrI9j5<1F<6=Mw=+&{sYcaLt0S zm~NghSm?50MJz(EEhu5Eiu4VPtJo#w|G$3z=UVo=H9a0}m@G`Iz+<1M_uI#_?`YR* zrPxfX#&{_H2eye8xn}9!DM0V(JG-QR<{5%D2#GM?v&0y@#9Qwy!_6Q9veCvtiFO)t z^%rq2n9%BF(nR}D9|zu*ZvuC-)q94EXQpe``-1}~m=|GYpw3N(l?rwJH_wdvTycB@ zcUj~0M)e1eg-5E`cF1CBo{luZ+1IiI5764oAEb5=hSDD@wwSSXo#$Vfrm{8?Woydd z*%45vvA+O0lpP)sSHeT8$MM_}#&sTZNi9R|8gOO2D&)>J^Ox;ha?qyH&x>Hy_H@H)) zK?uaL1vFSq_XYKL0jeY7+UF8n(wk=jq2Dx zh>^&cM5c$Y zga}ifiy*)VvinJ23dXqb6YNfYO(Rvw5P^((0}VYXc2`5oQhXJHg+AgTIn+FWP)nSk z@tHlJTkar8=W;YI=pnBRN1Z^B+K7i%iMf(iqF*X;m5|y5zb6zd=vlMOWslmas_Yr2 zIAX!~?WagS%2SbSVUj^ z9&=P61LoW@3WODeas%q_d!bKVj4t=Yqekn;FiMOx(oJ^YgSPYKa99z zcWM}IzbK6<_W$jt7=!1E85H2Fu}n=xw;|H-DVUqRE+{bnFJ-FcO?aKNuLo%?L38WBb^6U8;&}338`)eukO~OW*~|9J5TUNp7GdQ<1sX2Ibkg5 z{Acl6?UGox4|K>Fi88ZJWWBmB%Ss}94MO>j3Gws&+f8ACF*S5#p`$Y92#q&^4crWf(F?d;euG* zG|E+>92BH0Q2Y7u>P!~x%_4z=jz~bntT0Jz&0Tx^Bdnda*2G{Od{miAzFHa=XttE1Oqd zN2s*=R1=>Xrqdvd^Z@sOQ$%%7nNWno>T2(%LdrT~yp#rf<$4C|Zg{HkIglLxCQpNU zPBh1Ikbz0Oc=Yw#oOWpq>$8u`9Jf#Aqy4{$Px3EZlK*idyXUHJl5)f}g+y!5i7PrZ zl9k!pJwxVy{%`(76OBXPBn@HMdGIAC9NxsS>U{mpH>VC%R7l_0E zm`R=TofzAf{YE<8++w%cJ!fQim_&qj&;bei1o`5W1o1lpirsp;zD;oLsSt;#wJbMj6H^sU_Z=)!7yLM81bbg7iz(lx33YSJ2keH>QU@R zbd%%L6LaFRjge!rVT_xi)Sa5-`f2O2-~{EMNC!1layt+{+m-pVsIR)O8PgEsFe5OQ zO-a#K{X^bzJ*JkA!HPNS&x0JO!z#H7%JN`F(z8;*oGH9vR+mvyauogsOkGpu<>eph zvR1w!{L2*!Km#c#yf-$rE&xjHqL{=4Ak8RaIWD`pRM_~75@&QIay6AWLrGmYN35Zx zrH9+l89eLbg|_QW=b%UDpR>ouK5>>~(f3(`Z=gNL;;w40CN4H|w&kU$c>#7sZB5P& zzGepEQ5;Y}#E`71F%@2-i>BJ|jmm@kr<#qm%Gy>WB`b;0RY=uCVK6Wv;DvKO^{bFv zALa3h@bmK^Z;lSYxl+|^a6LEO(Pp3$rwtfXp-K|XH5Z`q($?Kr;AmKz<+Iy5GYOAg zy?$q{%)oHy(wDZ~XDyLwxg;8!ygTc`5f~Hl8)gDkTy`8sp8sYtA1);*ITKS>9)IQf zi|5I0C<}yDv3j$|PxPV>@5`I0R2diG4M9kusI6N^gw#UK-ds5sksujoe_m{QYv+VT zZkRRM&Iywug!UNcP^q)r8~-x+)lNr;SKCO`Y~h^sIlhhSxNPaUc{5DftcuW_Kwb_} z$4ZVo?VeG_Ny`-Kif_W1ordFVd~1V%;XfS~bCo)cp{_1LCCe9cDT+|5mDGHUWx-ho zxjimo*>oh&)9n-c&589n`GFlx4^=otuoRAdOSHt+In8=^oR*-o&QH`W1QZ8%nT-Fp ztw%FrRQWrt^`HE#98|hju~Sec60g|*k?JL`IaXZ^=bw*aSzIQyyzDP_?<8ZYGb()PX>r-OO-V%o z>?|8wO`Z0}Y|}Ni=}VQm+6<5>0XQy`9aK);39S0+}0 zgFeYWMBw2&}3%J8agqNd#L zqjPP96qjxoCqN&hR`JwE9CCe}E`ZgLYKa8}UeXkg#S z*_Dkl%oTU0FUfZ~Q*oqy4XX{F0-&&`w8-uUY45sD?^72tY+6uROOo1@ZNR^m__)J>`mEY~ri^o}ZobcnE zfA{_7db0K$xVf)G*jil1-NDS^>7Axq>6MI&P4Q&kEn~lmli9yT!uGD(R?M*2gd$wA z5-Fjpsh6y)4(abgORK){XEkW(5Jyfpq;p}`gU}#E=Z;9eQ&JZ5O6*N^Y@s?>OZf>l z{|D`!4AFf#I8O*t+BjKDV!I^jI zYlDk~?L>YxKd>-x^-IPC-*<5|Y?2~~bT^y)6 zYiWW;D-oxN3C*htiwIH73&qOc82RuyyKnK>q=Bj z;PpKCNA1NJHM*%0MHsGEh+L{JM)7}WrZ8RirCD5c0x}{$^jDa;!!GLjKR}Na?gU4M zYC*kt3~!8$_5)KQ1e^|kizK#1N= zT7bBv;zEAOX<%97Xs)D5os~QA3b(%2Hf-_;u5#zY`l#@=V76s?VwA1pag9w5B4jon z1a0Bb#-8RaX04nbBjy1q&7m!WvUF6Jq4S!S)l7W?pJSxLF@t*E9igs1BcQ|#m<4Bn zOp8EocNPP#41CqF1|5e2`%I&ojR2MFB<1SKN3VB6s{%jq8%8Waw41p71&Oo8 zFdF>kG*aVAea6o^&RUg=mqw(9X_}u-!<=gyg_QeF3R)tTy;lcW@PkDVk|`cmN**h>x44s0g0OKMfOlK_u7vNHI^qw!fo0`5p=4%6h|TAL;F?9T0241_rL4PB(-rl zDR_=RB(}85#WYOOd70qfT#zao3N^>|)ORe*ru0|shYg*fir0&_&>V3>@+>xq27QK#%P_^NK)GvM=dmbihN%^*Ri zB2RNu$zHKneaOW6`x-s}PR2UIcTs;mii?%yh>tz-+Jd-Bc<9i{=6g9!>U?}HJQl`7&C@9U2!1;{APSNwgBTO@XeR|Wp=dFw%ouSL8K5&I#jL|m$=w< z98JU{#A}Jd8x0s9AfrR($$1P zvwPr_=H}1WeDimv7D`Fm=%F{5tjbI0 zrxp8rq;X|2GKv{dmG6vd5CmsVs{4Qq-B7m&ztJ1Jgrt188)sX9f}^Hzb=f)jo4dM7xcZvq9Wh4? z(_uQZ_(9`59@dc&Wg)(HKnSEgBa!!dU6>s5#DwB^5IClcnYU#tZA15>qJY!tbk(@9=8zwZORg=q4ZqRjttB%2aQPP*UYEl(ix& zbAaJvEBLl55onu+3|IiMEm@iAJxJ34V55Ok9nj$~bU$9)!f;D?W@b1`P1axQU|_&z zFhD%PC-^gbb&Dtu;PCI7T)}3lIM3erl`qeA2ZZfE&G`n0<@izlR95G|Cmb6H2TM;Rk;a5 zNCbwu=6ja!JsMBk3G+&#v0j4p3LV?_V}ID2E7|tO%?{d<7-T*37=3P98e<&V z5r}x645R=!YoRySo}&^Oinqs7=9U4GllHch${5+%K+-(b%C(F+pc9 zqs3XI*|8IVXc(79KlA<_kdbe4ineF{gTJooXP??KA#QF5`!BVCvuJEKm>)Z4{FrF& zDnyc&Q+*p6Uf54Lo>|^R^Gt0Ukq5jtDvB0ZJ3|Yd#pvueR)XDb*jQW2D|8?DO~O z__)f4w|lzXuc*a)Z~DQmV)VVZZravk74(4F(Lq`XKnA??M7eM8%s#D5EVHoz3=+ubMAv7=W_D_F=T_Y>R$YE} zd|z5S4n=`*&(|FG_N@adiuJRUG~b9XY&jyo_EfJEO;Ossa$Rb8l_Mt^!cF2>GnvBl^M>LScrM)^%Y3$`ps2 zim|48bYdbs2kd|Suh~#lJtHB(^>4OOo)8eMI&25d6fDOfC-kI*i)R)#z8895@pNE= z8C8fjKB?I4_4Msakz;k0wb#Xd1pN}H6`5SOXIMQd3t`aRs>a;s(ObwWRzH15__Wl}$qY9((5ilrPa|{R}*N z1ds-lsn&9GK)cE!e%B*BhR!dc=#y_3R8np?I=+3IY}hZln3W{I&|CK_zspSU-aO95 zzVyhS8qB!TgkOHF$B${v|ICHNsq4P+R%_+ygp8b)LOLirHSW|ZcXZIC{}Daaxe%w@%QkzZq@KBvBQVA27e=a*UqN?dp zLx>%=?(hsSm^!NKNJ(QtJG=Mi$aA$M$V|^QE$N3&S8iYD`aQzu0N2mjAiw0J=D^Yy zLsZo2vZlhu*H!8#V>|s9L!RKwxT2zxweCA^IaRaK`1A55AKoru*K>Op*}H}?PvAdb z=8N5+G|u;s=l%)#^vJuezN2V|blh0{!lc8>(sHjKB;aM8J=nGb&`k279xwAPbEt<* zAIXGyg<@^P<(Xr#*F2w3TZ*PqJj=%r5d*B)YQoG0r4QQ8>ChPz>k_cgMXo5pW6 z2a7Nxljg-%5iC__V7zIUualMR_=1x_07c7ue%B9Gn(;1u3`jp+^jY@!xeDRnc5k1Byp~gUa70oXU_86Up4^V(0Q-eJ1n2qjMl+ z!g@gO^ammb zOZEk4V&^IwJIMEMRj$cch*5e*Lg?*5nLJ3a2kol=)_KP&fTW}Ab{y->jd$;G3 z5G7{=WCcS%ygkq@@*u&_4*C?sj{AndK7710!n4ncy(L7+sIl3nRfLHchkuwLtABb$)&t_+x*A;EQOk&X?WcW6Fn~*&`C3c zbFbxJ^m}230J!S+Dj?c!=c~e(YwN8>5HL}y>(LBOAbl&dnau)PVKA?6xBHB>z_QCG(>~r}bj0AQdzAIA-sDwH}M1S z@^CUeT;6%shSt=h)nWk3<&5Rni7^FJ<5@kEPk_}Zo1P6}6^Nlh688h(lySLT=&q6h zg`S&-VW7o+XBg}s0!j%)JTI7>ZFrnXv~$B@uOg`uu5V5nFPL8oz+4PPzcVlE7}z)- zee~Q#Jkusdr~(3XgWy>!T{-)F)(q^W5hG91XM+Bc!oKEtbxnn3SK&9#A3`;;A;l0? z)?r!MVL-wD_t-CTsxIRf3Sqz{kO;yX@G zjIgKK5Ks263+oTwKzg}TSWwDvOaysJWu?AJm(B_~ZB^~lL3&?7eLnB1wK%)I@5wo8 zsc^@BDY8TMPt{kDL;*q0Iqw~sy$`qEQYRY4w!14XaDDlqh?pi#%>He_r1gaBHi=-` z)GYogH#sL|LVKdr_v^*tU)eFL$<1uRe-l=126D2p?f1Pkk#Zx_VGj@OMnLhA)$xyq z?o-m7fcHJVl*YomR=bgKaWJq*#Kcq~=!jN$GbfLI*92CO{m8(6X3l`Vucya5p9!GQi0fO*&Qz5hfAT2mt!G|q?L>;6(8Nt4B*E8r}EJcX7}XoGwlfmxrvE-fJx|a%wo2T z7dQ_pm0M|%y28W$o=puyEFnMQU_1&+#=e#w*KMPxni>n<$A2%DF8ut&;=|8qY4`U? zD6Or5HiRfaBxHS~+|R)sxhk?;3}sW3DY4f4Utmje|MFc*0i zXX?7Lto+Zlw)E*p+p!8&NF2f%51K)8a&0Vm0Pm|Kt!u6dv46I5*Hr@J!|RhTZSik%iC}hjU6DGyqAzWbhJ+T_xy#n z|LYAwvv<}u{Pm{*RpZ-anzHfUn;A(`($D8Lnha};ZFg%5ybbH@J6&VDg}wM6dj>nevnUsQbFo*!dT5O@fcM4mBtTcq8j zwGCiDD$5;uxQ0jRZ(5pplnuVM*WV*_(=kREgK@+k(-Qh0^<7=jOYU%*zMOZPT7sn! z+rMuK2y>|DXaG@M${cA!1>BNC6-jj)4qS4-12yB2v*}s+mI>zlxFW2RevqUU4|Y)r z`JRbBy}M!>yLtaJQl1IG=d;_u{;YVE7DIW<>;9D0{THjHrKaTqA0y(yBZH`rLF;Bn zaGEF#ZpPk%!+<?QzqWRxI(%g$n7AQLL{kM5 zQWevc_}MQ9z+QjF?TIg~H^7JbsZ4raSh{V#!1%z$uPO^RWBL2?aC<~!EM+krSg@v|k7nXe6M|i*trXG3;o$0!3;_3d)&B3$d=~)Ephwhwm%gcSa&X)d z0o?vI9S`!+=f<>ydXNc2gXP=%J48Q-R5b*Xmv`v8p@m#H_AK71`*C##B%jjZceZhi z^Gx~vA@_OHzarD@7TzH8bO8kV&01}WK<3jEGiw~fH_0$HX=wizarjMw3tAH7+ z^*i{s|JvHhv=SDwo7a$#S+o^}u+^6C_aa%9>v=FS&h{KEejX{@ z^BwOQNspUfIPxzQW;u`z8jIOQN_jB<@Lf~_O9f1_R|4q_ctK4nqusBFLjx&2EfDE`lGCv-o_QE@lUSYO<+`=$V0V>d5HnRfFKI&uj?O z>GOU6V#W40bU8D#qYzZ`c$nxv)QWHPGdW4?Eh=zpc-ZTkx(g+QvU&n{t zw0~?b2)_tP@dyonyS!F@7L!4qJ5o^hE<0PG?(ly&TU(gS`PkYUhTC8v9h=LmaiWx%G85)D>^Ot7cEL;VuuB*-UAYEvXPQ}CZJMfZu z<46w(8G6}sWEEFuX+@Sh)@&%^|K;cq)j!PkD4Pbc&9(NDFvt-`@e{=)8h@s)m%`)r z=gyA#7)m}LX_P#-qlRo(OV`=C=HrM7b?LI)V;hWmv_TKdf}*$#}@Xm zz3|rSJ5vt=b!(9Lx93|PC!9$H6O;_2bedH4$NQyw@wIFx>cO6^0#B=IiaX>G_~rEI z5XMCVC0h=VZs%WK&d(RXgX`AA?GA7v4Tbz~@BbpM4GiviXyA^Gr1eH-S5-N6-C@P@ zTWtWL&GCLWyHt?=IES9#nPN}>`FWS~MF5{b8I5fA`)cpM0LWK9!O9(gOH(PU^L(7X z8^SHQSvjVU6qQy#9nY=BJ{2g}d$YjHRi{NnlnR6hvQi1UkrsBYf4g38nzQIozKNnw zU4OXyi{a#d9rhCB2?8IKRhEsieaOk}t)xGPly#I7LkoXrBqb4I{!~(qHT{_CZQmbE zK3D@lN>#i=SVovy&*p|WXSUzI&o8mx7$@1Y0zhEAuUoYR!EUmwixr_i#qN6gGJVWKGOHTPQHN8{X%oiRBp zA=<_NHMKxuLWb-1k;oZ=%~jZvvorf_EH5c3+)mIN?1m4*JG~qG-(Eo5{`+W9gYa*a zV#Qmwv4H#!4o(gXs|)jQzq?&vIK$o>B_<`gZZTuSVcB-M7jE|I?q(n?Dk@5Hs=wOm zxj?ZR`dZCkA^fgLaG`r+MfyZzfdXLGeKs&Sw~@v-Th~n5Av1b<^V6?#^2p2U>^Kga zi_25Uf9L6HWCea%O|q-OmH3KkUA~Ly?d#iA>5bU3uIi@VJ*V5VyhI2Od7esUN#;C~Sqkf+P8j6|yeu zz?uE?jtp~1&pvn=(cxZGS2 z74Y8S&i=_}`r$ve9tM|6%r4713zoV_(_GcFX$6=1Zsd^Y76RBfqZ?nG}aG z2ZPUU2o#^}Cy(}V+IG0OD1^UVa-law8s;bGfG7?v>GeanOjGN)*G=9pw$UD({{3}% zqC6G#HFiGYwdHr`!*1W>1|agYgVF(B+~ObkX4Xmgsp<>yKRQ9)?&mSii^eOorGK#n zrj470U3b3+9dmz#p6bNuQwhe#3+(a5ToLX=S#5nX>r7wBrESLA;e{qIX$Rgv>Lf$N z&H6poC5yN}tTDxWdChu!%x1D|anP!~GX%byp&L$7cosrcm$&Su`<&TtVB zOSC!XjU}R4ON^?0cB8g#RI(Z2xYXV893@SPU^3vU@!$P_$>}bCkNjd^%KS~MUyHx9 zzhUjUzyI*u@N4g_aw$;rr)bXtVLo9-KBFwYg4m>N;Ngumi(<g zu-iBbI-Ta=n9%>~RMgzk+(o)Q<;0%)zIw{6zO%voW{||G^TiWTiwOPlQG5x2F(K8I zzl?$fo8)sGLl@cPJ zFd6b7zeLK$=cLAUQk)d{6vIP4ZYMrk*afpoU3Pr*KK~@Adp{p)jI{R3AI;E*9TL=8 z?euZ1J!pz*WNQ>h?7r8aCWqYGnvvmy9#k}LK{wjYsk-Kz$E>2crKR4@_3r*v5HJ>j z3mKX!cpLy!y3w_oYm}fXlGC`U$+9}UE+$($9|CdJY*T=l$WN#>Vj&tje-V{_whF_Y@2~Qo<(FP4R$Uc|y(~ zF3|e=4-e3gjN{|wR>Klqk7#*0a4Hd2r$y}ch@%V&n50AOFl2exL?!pMxO>97Rs#! zB$=yphT@+-r~OF?dIij`O6Vb#gI|O54}s?D2??CVVU6NoBl84Xp(RPnc!!mdG}mzY z;*il&{i9Ie75CW}xn7;8a-YQ6#_03N!}$E?>};00lJn>C1$THA-(R3eHvHH&TF_G} z_9e4h(b~_^Bk%MIAS43kGjC4tg(hz)5jTFc4EEIxnn6CUPSSVX&jAznwSpU^a*EUF z3cXsE?HdiHv!cynq3q2FcdD=!~gH)7s% z1ZRXcZ((TxViX~6$iz6>EcTrFc*_?`v5d_0sG`1R3^HdMZ*))Bpyl16a5UGDLwgy| zfC`CN7$t=~oxD|@>YI)djy8W+Thv0Tq-0&)Rv_S!1eGc&0}zxj)gB*{Jpp!hTtbxy zXO0hwABTPk5|2*Yh4zNr8(WyPs||RHqzeh6*aYy`f)55n7@9qOJo(U$n)w!d48>#! zfy!e2&-VW6q0etEG};YA41X&Mi|__5CFIXs3=FiwSA}DmHXio=-Ivo8S(@dQ_8@O8 zE{gZjSbVR!?imtXIy14esmJUgIgy@L9BPhv58U= zryw%uyQDe&-b6EZ;ikF^xrXblVbE8udGcn0k6~S?|>E@oISF62M?A8nuPCPSH$h+qXBJcsSD<$^}s3BkI#{OoOt?ts2 z_)^9Qy0=jVGV!=&0Wbr>SV_`W9o7l-EzX}{S^)+i*wKpjhA&K4ha9?EI)D5?VV6Gz zh)v^aWwwVD&Wl6Z+KGwsK*s&_Xy4Y?5udyg<@H<% ze@!W3t_d}vh79+GP54(_90xXg(iZUWeQ&(MC!H7pNFI9X^`~b|ucmh_MeDdc=-@$V z&&f{+HB3&72}_S>H?_EeHH8 z;;u}>M}t8t!D{;ggqCy%4vdT()4qbz(1+cj*JRf#DQV(j-u2~&fsTSzW3|-mYU(6% zc^BfTV&+yap!q9yrumPBNLmSxf%62WnHmW&n`3ZoCWjr49=4Lwjxv1_d9;@V)MlJ0 ztkUV6_bRA&U2ld%a6tdHg(9+*cyCIaJ|CFrH+e{C=xQ*&q8Ew#$l!&^oN22?Q~WaD zsAIM7jIHq)iTvYC>xi@p5WtcBe}ppTFf}S z@%Qd%+0P$8%#^6l#Ay0w6e#8NE$87i;mT z6DU3UJDP@~I?2u&JxHoSlC~EVN}U+gVc!mBX0|qQ82sEk#W{{*bxIhrc2^y(9Z&|m zml+=8xB>wopd+>L6m32YH-jXE9_dLBBmb6D3J&(;;;_6*E!Q*hqV*ISF= zw~5Vg%Erfbr6!>{K44=-gNgc=TK5KQe!nz6b2j&K_OYE!?i(1;z3C)BMUcc4UuM4D zs=oUeU(7OVT6B?SeA2a>6`!!6iqq*#P3&2{)~9MvkS4iJ&(2P@i~O9OF_hgI z@sZJ+bCWKOiDMHP*XBrAcjH(3%^69NiEmZS7%dT)c6HY3pB@IPl4e5Xz_a+bL5?-E$(E@is22^9tanu`*DAMFhEi z4ehM{<{PI3KRMVw<*T!ub5qjQ4breZjh;TNPd$xz?Z?DaTpXg7gfVphJ3LUBm$O6o2~1Ikh**g>W4RWX+~p{`xL!iASA6e4>#DJ`-z^1`k`kvg>lWmZGdr$2 z|Jgc|l}qGhWmDqXD0_Rqe{vfe(~3&z($-2N`7|+_SX|F+MQEjBh^%)FP_y6KAYt1% z>F5NwxnYLX6i7L$niz@n#Snd(pJV6D^Gp9>NO4v-|CW&-WGNG%%hOd85zn|_6qFTQ z5+b#gp7|hIEpqwZUH;{>6X`NX>h*b*HuT_!(273yEFsJvbl|6{r_~4!zwBdr`q9uH z+$DwcEZ;F}EKC=IO1G_RW}bknqOOj$aF7tR4z=WfY8jh*l~&}l2nw2e)+{tQQW%m5 z^YWq=qvW8S4RJBqRYBt{L|BCuN*L+Ziy@nBBzo^v`0Of5S?{~kg?W#EL63QytJ)12%HYMV^&h z#4fF>pk@@^S>Eau<~3zqJ@Zy?uq*lLnL6*njJj?vtxR3a(K*KWR;y3t*Ww8nF#4yEqFXRIaT|uva+JwfRc)e`ppUs z;U4!Tt4=O?x9*#vRn9&OL~Xn$2G3gYWS6ogh;byN3@ds_7Q^)dR$kNJ^?`Ay|4y6= z^JGxQ2#-57W~jeQk@UH`qyavB?h5NMNYCLmmshhuPupvsqWIh8a`es?mLlt=6)-V5 zA$N+R(JMg4EXY<~a!&Nl+Q%|H>mQ1M>t6%<&6)Sar6RaiJ}I&<$E4H)W%)4M7D*{- zG@Ixp{fNqmRYERvqv{5f8?77U`sit$H5A-p1O&*dych1izHsO(T^~B@IA4&UdP`!| z*Woq#`88bcwP|r1VRr<%Sqv&X*lYcRj_WGNO4HO&N^b1Jw#{OnU&)>=7AjP<$E%eO zIhdC2 zZ%eHhwg~!zrJi1xm!A#GSc?<9V@zLIO{Q-_OY1x`!}e%qf$>if5?uvy;lmhHcW{cJ z*FBv(Cc$dZ1uzMVO8=L%zy1Wrf3A2E{~bsCTtyOnk;R!9Ebq360sn^0TLL2bAaAl} zxnhBP>T2n#xGs5)T<~yNc3)mzuCITz9*L_4Y~Q`YLNrbhJAt*r4{lsOf7`EJjUmzW zF+ATD(HkXI9jMOr(xuhGR{6L5UiBKGNB=tuK%6h>n|q$_P^^q2sw=%e_q`;aUV3#L zKiXM7e{l|3o+=MS{9|Q}2Fz{eXMd&anuX%6?@Mi3+aWcJJPZxKYeNDP zt1m&9PA8myfAexM#83K|f2AnyRqAoUHavgmvv3H^-licw`LN4N7 z6(sdfZ;HpV@z51j9bxEtI>W{N2#Y8ewLivXdOsbwJ1)4QxUA7LTd*~Ch}_w@MR{7< zMw`=xlAM{mzOgLn^0Ba?eA_gbb$Yd#g~`muA_NFKPeTDF(U%O0BbcfhS#oUbHN4&T zSB&X9b`gUtIkjt>XGa&7BugIKb;Y03y81>^?ltot{H~rxwIj{!?TP!c@2pI9_^hFr zNoJfQWj1$IJ86Ar#E)pb?0hsd-?ZYqQNqx}`CUD8Z^fm!ElAWDAW;)d_deGn*<6!T z6L$LfD-$!c(wH=$MH1;ys=q^CEuea3zp*n;8^~B~peBIylXFf^_+6iOZNZHUbzRR? zSMpNdz76ricQo2|2GbQrh|rH@RCq9{mS@irO^Sr+x_+LeE#g49R+ zd9h|uY>rLL;H zHXfW?P_aaOw}>pY=}ATzKV%2od#^4(`C#ByKp2F)#iC$s%5i_5w`Tpg23q5#rk&w- z0jFcY#5yRIKK};_^UeEkT}B%P@;#&GUCeiVR*`@P&~;;GD|i8$Jv`gZ^zDQ#ysOGS zPdjox((0&^r#F+;`kexbj%`U~!ECNy?(*`mHXUZQ!a$QUDmJEC<; z2tL!qeL!7L*yNo?B5~H)wLO+al2Co!oftgGp(T^+{Vv$h;gbR{L#p8{5{b8hG7cB& z&QR`S^P-N`^-}J;*YsiNuf8|GEH#ZP5xOLLCpkgb$W)2bYNfWKD(i0x|C-rkI|_yq6yW@#Kx!q zu>~XO|BaRJ4(T~HIf!|?+n8QuOa=7Wf|pn3^SWLJLzWWqI)SBS=Q=6yatUJGy-w~x zGxv3cD*uK*>bi0#0eKma#~)hZZ*uB)j*#I933;VKVXrfThBZp@D{6u<5mqJ^!GPUj z!1!>o_Hc3G)75@;!`b=oY6Fb_LR5ReH50t!cRIN57;uN51rJ%zUwM$-o20^toS)|> zo6I(hWgDOB{Oh-A73m#8>@v+tCTP|jN2S_C`~`MZdSGAWD^LEsgahsWmxNe|%@Xx~ zaTf}5&iGEQ;csT>PdnHNxRG9Q{;45N6Y=qc${kDAA$l|D78pOtbhZSvg2l3OjO(1AsG zpU%!ENnh~rS4Vu~SKtGD=Sr{ooE680!R8F?yatbM4i;Eam!QD>EkAD;v-a%G*&ZKU zBE(GCD~mr=IzmrZ*F z%%oG(NCbv7J;fh^o2KWA_7t_5#idXV8GH!5 z3*PNruU-+|d1xpY)W;+@u4~PI977jE@a{Jj|3Y9zw=gUyY5=)BYqi|D9gA-&vGJx1 zY(hfKEfo*Sx7fa;J9kcq+F38JP-|=J+3kY5M$>m+e1oEND4IK9??4pPB^6air~aC1 zx=T-jvy{h2ri^lQxQ_(0 zb8er~GsNm^s}9w`^9x$%VpPRvhomH+1=YFf#8Si#Qx)X5X%Q#;TR(gI@Aw)~X$lR$ zGt$+0__z+h0OoRXQfX4<;~NeZ4k526`yF^A=To38S~#iZiuK*u5D?yTe(JbBE^cRF zW^OI5)M%FYP62-#ixGnvi0yQBij=H#L58w^GK(mmVSJ2cu}^Zngz)rM!ia}{x`m$ zgG<)>@%J5GYly>E*9E?Mm5d|6EsLAv4yMEq(O4lTiYtw1cwgJ+7~@0Z+)Z+fi72th z_tVjlL6sV|_suR7*#?k&12s;8@jYIQTzE7XXI?S>r_)i)irAF6r;NB%WZ|5Z?Rc|_ zuUx-wj=TV0-+R8Ir*$V+lU(Kbq@hG30F)`jyLx%wXWzhc_QF17nPs;c*3=q$z}1De zHnk&)T9i$EF9>zZ{CN0Q#-5EEz=^mxwQgZj2j3a0a^+m#*<{Z(Q7Uk#xTla6PMjy@@Ozl`G-)E6=-9n4C(4= z{dT^xQ&1o`;xsq2@PVzxeu&4F77sZt=(@U@+TSm7G(r#&axW}0E-~+YQt$9swyGZP zcCDY~%wDf{wAwt)pSGpg)UGY7$ZSWwR&I1tj+y8G4#Mbn?TB#dE;-}+EtL#qcv6U| zNz|_I=xA*ah;Q!wKyu3r!FH>u9YCGGd%YWBKsNIr1g>T!1^yb{ot*floZw#`9ty*CZBgRzRG}~Vf+MG^p&n<`xu3w<5Lo=OpCA{8(O4;oqmO@+lg-BY0~3)?zXG~6 z!GLYCc_3=EM@+1EQ#bdPMWrvVPa%$_%qHOG_E}p-Q^Yi;ciZ&{kB=3Drq36EernIx zbU8SBeH3#V7Gmnlj$F{DE~4<)ADC+Phz!ZN4m1(Z+?5wRV zoyQbf$pB#OVskT_Sr}WabhS~kQK)e!k?`{Kl1PA3DacQ}5eUjmhABSH%t$(0e=qBo z7`sJ}J&e9m_(+OhY-s^VWw)z$O}?iT(&LbAyVmyTU6_Bi@OMBDC%VU$Y-3IznrryN zQTM)VQZ}SxTCqKFjDlu-DF%njSw<+5hgHZDurJ%XO6k;8*eU%h3yZ_Kg?gUvEHy(Y zKDjWXsiB))SXt|F?umQ7wW|8}bQDqjG`zW|Du~ zn}wYvLbEC~;qQ^H?e&Dc0KZ8hD1e6-0&oCzR~ElwHMX}82e1~#9}F}`#RWxho;80p z0h7BqeL27lj1%&d%EED|93 z4|EQ6t2=AG5o9hcZL9}^MM%7!EO6nHprR{PMIUozsv$t`1k*xJEmLBYJqv_V(v1gG znz_r*0io6|M16prNN*=zBX}(MuY&Da`V}zxiCXqbg{u(;dR8&Y!0nAk1e84&pz${N zd=O?+$3v*&>psOL{@;bOHTo{tn%)`{*31dRBu@vze-HYlS(Q5yFFU1_M0K45( z^*mr$_;!#UL>}oce{6q_eojB>_=C-;ag)mKHFYtnw>cAmIJ#l;tTWJ9_%VoUPC+=u zwDL2{g)*Z0(NM%j&PBln4f?%gYJAk@uRwO%)cQ(O$G}eO&!QmZimj&+H;jxq*o*t! ze&XMaFl90-xR1-TkWoYgNIl3iS+_VAI4A7X`U>T#hMfmwcF)QJjl=Pbd_)XM;C%TM~WCe}nJYL=izqMGo)LqU~x`{Te1A)|@+3orwGcc$q+2eeg)Gf=a z;%3#;Rpy)~K#7esAz6b3eE zH(Hu|>xzdgW`d~;5D3HB8+V)X2vdC! z&M|5YL`TCk(CZx$;hKLCT6{?;fUXFQoF`iHZyZmo#9F{RlLLQd*Vr-*Yxt;r8 zf~ltbk8cG2(Zf&E${;EGf<&8N>XlyvjW_J%=Rs^0TJQN7UhDJW{-DY@=UM;XSpZ?8 zT`d`<&IpT}DT+j7p`NKd0%}s|_upPN<$s|`{uSm>SE##tSyi6BwX=zX12BGWKQWHJ zC-Y+4hEwz&0ijWHiog49DPCSrcMdYGC10cNwK;+%j)b8&V19$0JKSmSaywKKMAoiwF zMq9u;NeIf`405my$ENKmM)dj_SKJ_XpRz{F)|3G%6$8(sw-2Yv3$BzE{|4u?sLGzA zt_HS!c8M|0g4Y*50w*mj#ZmAJL2lm8-454C(wK|SEf-1r9h?wv!;QtT`$$_O-;p)p zn1fiH>ns6;mV+Vk6AQOc8xW9B-kAe>Jf$GaRu(N*rD=;WBjJu{wdwja(Su{}x|Wte z`MTcx*R@3F)Mvm_FeW`K0Pt67eZeX$D_%}Cf`Uk`hoE)yLOk;8-!}f^ls4t zr%*Wtx7zrRI=k18>ekv$KGFA{TX0RHIUuXA%DRH|`z_~E%m^%DlOGUdT>lj7`MKea zBR7JuX|Ng|y_jh&7yIk?XlHL<_;b%#Q~B{y&f`qirK3!HuU*9M0|p|FQYijhI{72c zB5zXzl;p?hwMHj-fbZ$QR#vfAr(G!}y1F+|spy?jP|(P^i4xvs$<1v?FOq}Cks9^F z9{bRTdDQ@XhC(o%pdcisr*m*GuCejA^Sno0XXg!k^B8caK)R!}x27OjVS?<*zUz`t zSC`tNK)WUKb!cfCRaq)u?PyKDuCDIzo0w#J`r=XVnTgIO$#(LQf>^g0bkiho_wkBb zDBGVU?1I;}OLR9lukvJqZps$4i;@I8C%4F&UjWbPZ_M3n zov(#1Spza{bW?Y4m)HR3p><|HuEl8PlAB3xpw}|1K`1j&XjRwFSvTy~($2kgeXY6U z_(u%k1C*hay=tNV^W0|t0U{qCp%LcFH)UdS#$cE&=)I;Ch9wX4JzB* z`XU>;mSw1U%t!;^w4sTGgUmKyE_0s8wkV+`JWsP zi*xER&P6vgT;=gspQlnBl^<|7(1^s&o7$DoXuwKH?^gEl9p&-#l7faB@uxm}TOx7G z>aPI#vKI^Omt>c&vjSlyp;np}#e<5Y<)b@ZTEqh;95H6r>K?-@w=Nyoqfa`% zC5=!E@5HS4DN@#EzJ9Lnd-|QAwnGJN3mvC!WSjlN0$6JUqFZ-BzL#fEo}auhNMx<8 zLpKxA0?p$&RCyTQCVV{_D;t0$`o07NiKyKS(}vy&7VQ%zZ2RQ;eEX&&0KzZ-VW_bf z+cdb=v`7!OkjF-BcUnnTyd{`b1@_6qm%8q z8@{f7{^j)`j8394I|HO*z`i$3TNK^-qDTKf;}@6gASB!KUvJy!Siji_;P`sA&3i02 z^`j_?QMB`!DxUqs_^ZF4o4xsnZNvT$D8ZX;tvcIS7FmOLod(nc9%c-<%K?#U@cvx5 z7&>TK?Ks~R5_`m`);rdJm3#-*Bvw%P z@dzM9B%g5j_~spaFN@y-ZV+&A8#VYKKt%5^Tn<;MzJA}yzE3#H6!Ydabsh_!Xsp|wR{K)*wus#oVP8`AN%=j9Am-!1A)~1oc15=izDr&gk_J+&qG)aLYAQr! z%5Oa)_=j?!nf&zh#M^&hHr|gHuN*nzWz?zU8QV#39kxU`O+S!l2>*_KFYF`EApT0DhfBf^@>0QMInaft!n}=f9F4q{nBW;j|OHra_j58XS+b5 zFj6y>_IZ_Lt?gWMf8-KZIUM2@Dc_0V(P?jUY!-eg{B$T(APoOSVH6P4n|mTG%GJM~ zd@wRnUVIB7{Jh}_EA9ub@}PV67~b+Yl^Fbhhm2=kEyK`O3?-j&m5+ei6ZyG3fp7*9 z(Bp&g2T)}}@lFBA3Q)I!8TYZF6`|gz6AH*H(0+0I!i!_P7`{WP{7D(tcMGoC}}MSc^tg{;%tIGz$L^V@TF#o zCjn3#C26RGy$$s@b|Oz!*G@Y#zf+<%MkJ4I)TA^ek~s7+`((_{iSWIl8s?hLMr;6Q zp>o@bBN3mi{#n9;MKC3%Wp^8F#_@lAOULs7PX9qRTw+3g$CqOEHJpXtxlz)* zxPD!-pyjA9cpml&B|Cea0JgRpm82Tm^)DSCR1~VJgd#|dyTGhxJ3W=Vh4kN0*wH8Z zT&WlEXcts+iE2HF^W9_caL>9-lTJQAZ!LaT1n5j#_`h0!G4MWo*W;S%N~Ay)XLY0X z=6(}uSuU%@dy&qf@dwVCTx1i3Ym zT)7D?IXdu7o367|Uwv`#-s-7zgt@)<%GPHb=gxE6@>@5a|9jb&#i+j$&wTXNOMw2E z;9CH}M-xwb`+?asH;rwpPn4mFNbW1)2gxf;gc$bPU&u2PCrZ<8tML0An33)GcPG6M zvzb39qu0QSmDreHfWKzxMoL;Ejd^|75uno)xt@5!11L44+;ihW2Shx450tZ)nCH>G zzAR#eQNi-tuiw{vjq8_OW5di#H6#=vdFP8{g@nEKBWzM#q9?vIBsg;0#N-_?ObL!PL0WvIDTrPdxkn=;c~XzcA=wZMXG+&D-@l&Ssjq#t9N=#L0{Es znA|irXVq7Vg8rxd@y0%FewiRuPJNX?Upk8jZbwqs-vs*aB?v9lY&Z4)a1NpjTwGm& zhYz4`0TV?i`o;8$KcGo{`mU=BxCwZg3H5^gx`8b!=?CPUci-d*Cx>;1y?tq$WE@yn8zS++w_7mi=IEw|QG3qrkP zE5Yc7K+g#Lji|05oJwkBc>J@z!IIk1dr1~sJku99(ecmr{_b3ugec6Vw&mqT8kK8y z?#pj5wkx-M{yvG4Z_`*&Y|Zi9yoOCoxar~#(#5zkGII;rYC7R|pS}|+y?I-^o3WD7 z- zo;mb728L1g_zAf{gc0_Vr*N*{`tNtpa}VRs-Jo~o<`&fzY?rs$@=K|r9UeJAHSW5R zSD|@8FZtjxFd~5&6ed-4cX;o(JKLOTU++wmo}t~{`I8c8o&MI+PRmN%K1!%8d1Feb zR{Ns_?Jt?`krK9hj5(>myQR?kZ&h`W@RA2*6%`fbG2*@QhQ`pL>N6j6d%Vy$I3j!XXnda8%6v+7`s#_sUx@uI@Q%^D#) z(pksgn|UafbdyC&BgV(PkDEq|x=IC|KN_}Hf8$pN z+!yQKXhF!vF0+r*J$MFLfpdC&-Q~>eY#Isaub6Uga24_tL-!<}&}37zwv z+nbvs$`nUkz&@p$;AZReX3(~_mVbw~9|y`ATaJO&Ht&X7 z0i%w&tP16uymYF=y1||!c2RW3y})-)nF-n7lZGm0-wT-!R7o6Noya{4aNX^luN|os zV#i7;uz0^^1wgt)DTtBSK&FZOW^f&p8?d`T;N|#rZGKV|^wJDaZ>r z`0LbP6Y5W9lzX&7_rx^7jQwRx{mDJkA!Ua)`9#|p7}jZ(yQ#j}+d~JE!xn*V5zR^zyQvu9v1O0uv{>M!P~4 zEUdVvz|P73YLof(?21+`zUhu)zY(gAj~IHL&z&urK{hsPXLUw;XBX|lhQXQWdm9g}tt z97Fzb7n_j#w3I2(6hg&A{2neh5tEAG6JG=?A- zeFEa;ky6${gAu!&9`5b}H0JSNcyQ;iwEB4|LDNxb+^BtfUpKRksZx|Tn}&mA0w9b{ zeV07mYxHf=wZpYjjS>TUWhMT%IU?Im`I!B5@18eb+9-*Q>8f@ zF|87~YG3o-`IY!eycySL)XnCxDF)UhFu?kF(bgRNqp*#`Ox(PNKO zHW#IvQnT*^4S8x(IC8q`|&ADR)bnz4;lRb`}qm%){c4H<}>u`!Zm z@0Dxp8{XmANPNX7OyvaoqC6NVUGj&!>KlJT2XZ@p^p%E@Yx3G!^Gn=ka0|cf&QCdTif!CpS=GB~ z>}E<;6uf#{J@9jDJoFaWEExy>z46F7aZWH7n)+MEp zc$a&#vj+u_K%!L9TXt`J&U31~xU*|-XHT1|`=AhH*mm@u!&GH$j_bzGCXVDZTp^PKjkh7CYk#nmXdac zCetD+W93|K-i*8wJfPTw*=0>niYQjJ=EwJ_W6G0wlJYU)RYs0K{)c%BPW0T>{y(0l z_@fk~XDx@Otd~ghq=fM!h`2TsBfn@oFEiwK4M%sWsjVtmYTteWY@6Km zb=qnFQ5p(lb3$xkI^mwAc#Q}?$bRjPJyx_LrhIlz#`>Po(9(irVq%@^1vsL;wbp`N zVU;%Zo6S+8(^5pG%sGz3>c^M;3UpW&Fq~75@*8!!4?a5MdMvGG`VABV(`##AsLa0s z*egqOOHNKscFswkF~`G+(mehH>T`w^RKK(Q3+QB;6IH{&|7)c^r(&aSpT~)4HMr0GM4{- zZVGbKYFb+%{6f5Z%ctfVC992_#cxE6DSSLOSu+XjTB%1xNMK$JFB)8xUzi8n_&P_* z*h*9!}k1obPlx%Qvb+G)En0`;V$x^<=G<|6-LyIzR ztHL;+>=fyy_R$3g3%F595fewyV0^dGFg1<$cq4dVQdUzmr(Bw7Psl1%zzn(>3N>%$ zAq<4+P@tiPbW1~Tn7rsU;N4A^!Cl#^P751dKPM-4)y^hK>Dm3O!PI{yX$53`B6Ta_pl zSmW@~64`!a8z*VCO9K?>c1ztL`yy*7?h3Y=Jum1wAC(6M?RixK1$I{(pYBT1ybm{` z1)+et!>@hILRBD(KbkO3vWF(fqnFnu-^EL+cJb3KS3s<^Kp^o2EBHmSICPX(WB!E* z1X_t~I*`{c@VdKy6B%oC#o9FDk0pxL5p4~%6(L|P$w0gDNLz+6DdAQU**E#QWT3eH zyWvBVs(*KilTq3@Xa@Qp9Hnt>6>=d}!n}m!a;bS4dja-;%fZGX3fRW5&bjW9#50y! z*0E~lTRu+Q6qIMFhIbhtaec`kOo)yxCf)BQJRFk*20}Ikl*L?spTc)QGCd)<7DbAS zN~aV`Kse$&w)~PAEvG_Z47x%B$U^UH)B69#qf9V+ER)b0mzzt;F{{-;~>YyTq>xtD6J z{zVE!i6&j1)(@D`>#%JTmmSmV(1R2f$b$v3M=rz7^?;5{|IpwE+^EQ;>1=|+Xmt0e zcScZbYlzdsXsaW=Z%puObNLo~{nf?CKoEwg)}Hi&!E1q#UPq#SX`7*wbh6TZ2h$gL zq|9P*#^;inOeN0eeB2gNjYkPFKEs_K)mGwZ5mRBxTv8T>3iw*sz>OPV-`z!w5lM)ar9X9Y2R`_CkN-UM97X)a~%x8HVfy z9oydvpSK+CZ9(BJZB)r5OIBxI9-iHC7f`OH$umC-I>#C zBWgL%SxUtC=+MCDC)LvT)BpRwkx%X*Y1~J-dcuAQN5O`#AvG4G`YyUS_A+@GGnM=% zzq`#t0{I+#T`*M*{DVhYjhmb0M1w9=CRs_dqDX>7vxkU$c~|lUVII$h;}53*nI40& z;F1^KEFMVr)Us>%+`bY33y*KUnYQy$YhI8TN!6#WTSx5!&W&jGo1|FUm&MY~62`V- z=^wdvqZ9!#QI)z6&F62irihj3Jg9?R$u+qF>!!$zVzX`9|n;5(z~} zKy!iC2l7uUNx_lVTXcLyrH!?Jh|)|VOYsr!R=fJe-Mdg3;j!1|{h5Q})h$l?Rt%S_S~hqtlI28ax(ao6d!gkB_oBs zabIy7+mlKb48E%sTySC&v8-W|&RK|vmD*E)gzJ6c8@7!asj*u0pr*%>yPy+h)-;rR zF(qyKEjuRhFB2eqQ^s)e`5eZ$cQ)^9NIBdIv();*G(e|XWDoIG`9;lqz6ti@@~)`O z8z8vKDV^iONL>$(M5RV^&eRQx*#ER;2J2z3S>ROHsNlr|ewTM6Nc_r@(OV_x0G-zi z<|jM)`EjXSQeOT~CYJPG&QXx*w-R}@l@EcTd?Z^S;bJ-*Bwl!8WKS=8qqfp|pRYUS zgO0ilIQ?gdo#-9$9B1L`vx0mqu2Vr9g$E0)9vJeMQ*?)Q?9;EtyIdPN5?F{ag@xK)2=l2oIHX_O@ib^qU?6r;I*cQ z{2f{HX|we?FSkVkmYy)S97NceXP>is#?MZ>h8;09H7wP7YOv&JlYUt9j|Mxg&?o?z ze8&eqQzu4WM*^=^M6A~NQNr)2*_=LWr9IF0Z@m4uO;i5Ml?5?1E5bGN?DV(~5?(o# z;J97XJf7Bl{W?wjxRn!v=e_FjWmV*$PY7O9-Zw9^p;)^@DU*6mI`E{;$PAf%^;GpO8bd?OS)e#MUp2Z%F@XP@r8Yjre0-V%S7lMEzD7{~@kTO$^N8 zLfkh6)>s5Cd@}N-)Sz{hAY2n1W59qC9UMu;;eI9@1>kRG_a7|Fkn+(DA5Q1^O588k z91r+Bq{$8FA*M3&AU=;ThX#GUiGhvWbMg{uCLejw`X;3ZU21}00QMPW(YBrj<91!0 zhGfQkz!WahRTNA2XZdXfuZ*e4DlJTBk$_5p*6KybzKUrt5`jQSer=It{R6;fme?ku zBBVhN4aNJ($3b3ns@BrIerb+yA7bFR@)GC<`dw=VgB3i#C`ZNHQUNgNh~H1G^{uM zq?%HW&=BN))3Eu^d47~3Gt&Gd>E@ zWdtT<(HhhAEHf(7k1t9(-=CteFio+vPO=9SY{4#2^O2by4b8GbO1?nGZCqNHnAryy zIEyv?ubm?f`0!ehmW8~0vIyzLR4G%y2L#sW<>jV84)DnDclsK-RKF~iItdIxhh7gEhF=fxO!%b zu9xuO*aULYDe>?Phzn_t9wx>z=kl9{nL$1NhH}JU`Qp3P)xTu@riHY;bFua$vOfrF zqifw4CUi}Arce;B><=CMj977OvTD*SIo)l2K4i}?AFeT7URW=ERFZm^bD|J(vy4n`s`$)RiMcxW$4;8!$fM%k55-+y zm<=bK$01;FD04f1JOsU{PVlQNb?YJ-86$Rq7dcS_KTtC`t^L~UL!hc+uF`&*P~QiZ~>fQVQcP>WT9eV&3G) zXvNki?fdn@K5NE#wkQ;aoF=ad(@R6tmXH%@$^?`y8Q{F0k{2LJSi73?B{HnUb> z0;PJW0+72}8}^qK9>lMP^qqqc6N4G91!92+uZv=mB7dd!u5_Q$-pPVRON}=XePk-V zrFLBb0pB0}&19_m>zNtwEDJJhOp;>@;(8jSr=Ltv_9rXIsqEO?{P5J3m!|Uo+r$@{R;Y zt3fhO9Vw)@zgsO9Dhz807uSD>AO|0YfJ_JGrp{`-%Z?&VE>SJH0;Q`z{#o0E4KD3v zc`1VeLWtbMLuRbx8`(Ea61#+OUA}Zzw}y3M@Me9@{1j~=^(N4sLM8-@s-GtixDKuC z_~si%2H*7@N!j@zWw?HYc(m-3zQ)XOO-&z*5#(4FemSG~<@lG!M16Kch=?3TOs#W` ztm)AAl!I*aT8MxM2?2=BamdHT3=0|FsKrbECO_OFM;QY7_F<7A=ODOGt}om#nTf4| z*q4aTn0G8=jot)yK&~IcW`6guBJgyzFwfQhNNf>>3K?%@p^A zk)u{{7BZmIZobZ7RoJTWQM<@=*h*D(V*b9ESiCUe09&uUzVS-AnrHMU>?KF_@X6 zfiu600)m4G(%o%IPCL!g-rRmorzXvzBT>h`9J{3&xAtWLIL-5#K4%abalegHF&iU^ z55nvVKz7&lc#Liuu#?n5yz2h30I23v#)rr@Zbikd8EKf2fpdYt>sAxB%`$ zHj>BNFU@sQ{u;y|MEv>2!iRjj2}do*Mb^%D6ji`-{{kRr_{gV~AfkD09ing6!( zhp2?FszySGbACD~TlxpnkD-_@BPR6`r<8FQPQzVF(_Ov@X5_pqPM&|4KPwPZ=tYle zLP2GIb#&4{HraO~g*0>XjwOu*f&-n*DdYE468@rWm-bLWgm|TG z;8;qu?`G@P2lJgw zDiIvi7(^h7;GCm!&#~;A88uSKIfpVSe%v27aGG=n$OX4&eNd%ekT}*<+yd|!b*{C( ze+Cp6`z}%->;><`f}tSNVI%)cjU0xcMbe|qA0v^PMqh$8$`G?&=~U~{PkTh@Y200$ z^S`|LKHNPEKPN3MBw_B-eTGsUd#iwVu$aqN^*Cru)YVLEl>UPme!_2W$1g!w^aGBR z@Lq6xyc^wq@8AVLe1sQ$2N3%K5S|=#P&hQTw7!NM_d}lD39EtV%gZdl@{{8Ky)1f9X z=Fx@MR)pbaSmTasJ>SNZt(!ba&9kV!U3Rn_@Pkw!AqlitVEBw63!Y9=V*~=`9_*8? z(lt$KKc(=F=#F1;UODgkZZV2oE@OFi{9=o}hW7_uB$Ya2xzDnoyZi787TeERkcB7I zYTF`JQqVXjg$Sy7Z!HB4jNT&4r9SR1DCfv$D_htms>mJ*!Q z**hJ8OOg_C0gTT-5K*t!O^Y(TzC_BwUv*yY<2dnC#H5N$D=4JnzraWCsZEevx*K@n zPVx~Wv6>zgibaWT_Zs?}JuAv4eZoZjEi3)us2MG#fRN-$0LQr-Yu_Zvo8I>O>6f4t z$)%&MPq=3|7amR-DE7#GdyWgKLT`lO%1P61pJ!Tz z4?W7Bueg{&d6l82vS* z<|?&a^tH0{Lg5ZhQXLJ|e*uY5Zw)SkYgmI>_#5OhRi|=NPc_qso@9gr+F*jUKe<~? zt!>`3kia{`v&v!Kcv3mlthdoWQHF1;{_!EJ@4ebqyfC(+hTF&3Zb-ytm)VYSSs_(c z+YKLn3>c;vMEcY!`$e-26}*=%z|1RgOW!X;qO8tAepRF7`}lx6eiUp!r~m zX`PyV5})N&GN<(Z5b*N|O|PESLd}xhK1&is0uu)Pxi!kXT#c97lt)c(3y@07IlXJ> zH>n$L5u)Z}f01U33cqP`4g?SS4I$p90_`;B+>7KC1z6-j`G;iZUNCkN30%KuY!01dB-p z^_apI=HrP5*+S;;`dMw-{WP4C8@GMAd0WvoV5*k9=VR5I9NJPFX>cX<(fE`XT*%L3 z@v!^2f^tF0MC=z%tje;>Uv3nFTsk*A0ctBt@e*;;Ww(5_{Oq?srxs58+@h2Tirmvy zjnqN2v~o@{S8>gw5MwsVWokTfij;efib4Td3fUMOwY1S|0?zC(;l(+r96mKHTMrrY zI#ma9y}%MZo2|ux>8~L}8^2$IQ`Tl4Wb&6vWn%lo0dx?o<-@H{2ys1*qgnUd*!A-h zaBU`XN`KG`g}rlrQ;iKD^5lMbnyvWBU6w*W1=QQCqb--!SRR*p_URLD zl-9ug%6XiIal05C4475G>Zd79BdqbthT3!KN9ekhpJ8LhD?FYVtdIp#su0Iv)n4+h zoqTxOgPVHx$2-J%GIU*=^}xEg-9W1?lGc?%w@3@j~ke*8Xas?xFFWbJ^|` z(H_doVrY*jw&Eitfp(@}xV~Oy`Z#I_YYgSZ%*3{;Y$IKtvin4O_i-Qn8J`5*4d1*h zs&-2QGLO-8$4&id?P-_WO6^{s`<(!6En_8llYWyRHv^I#vDq;*R!I;Fb#(MS7%16mDoNI~7whvis;03K;cS0YB-bhh zA1gfZINVtK1q=X;w7RlPvbBQUYmol# z&@T)`d$Vvfm2`z8O^Swkms#q$oYOKpW);{A{coc@bOm6fe%Egp^TL2baJPJhO8CPu zhkZf4W59-6uOjdeJ37grSo>N=wLltHWjmTl4%6Bd@ka#V&-m2rz+n3LiATcUu9}}k zF$DFU+$${7MUKpMB}a=v-MXr)Vf=j*CL+CC&!0s4Lu@V8=P2T$6zP$=JpFo-*xVx^ zu0z=ULXS(8tk z<-wT5_;9E0ox4f@4ii2d>m~dM!S4fR84TP?pW7_vz2B8+5PTosGsQe-8^xhp3=w&* z&lILFD+jRSsz0Wum9?AkTy3PIu)ree<{>`65sJR@O!JLXNWhs?<8W4sdRYzZC z*WxdlTX<}f_!#!1lX`Sh?60c*bMV`loba%)3!qFpA^!;x8p3!HLnFfts3D*HKSL%r z96mHV)wo>B%RI30>izs31I{-O6)x=2nnQ@Z`T}&HPjoUmF-(egbWmJ89_}?T>GUcO zgd+0Cu?5Oz_Ry=RG+UWQ$+ps%_M!_+(n2x7Oxj@B88kn^I#!Pf#AWy^-`_$#4vp*t zwkTEnCP}20@~*H`9;gNJkBQhv(r(r6H|6G4RWc&SwqTHqO-w99RaYVvEfY;de^H*H z8YJ`>M7oq`7>mvvTCy&jgqf?FX(g^UR5aQ~m_{7+Z4=#A|ERd_a`dVxK8u+bjPRfifV8^7hXX>iMi(+cqt%1Advn1xudbqE8`Zd+l#Ibzz z*Cd750o*a!EY}!y&1JPMmQX3^Ju!ASeC*)x9gcZc^H$!Yp=@WTkdi{3`7}Kr2y{hD zp<^KLb;#}fN+|}zNH|e+H#XKF`%DJ?55=9&&Ib@2ad&J3<>cpw(haANVo(08q$~<` zwAs_Houxkc$q`y(;~!(3@d`s(P7KqD-#ipQ7ktYJ^6HOGs91-dk(XNMPpDf^Qa%tb zulpb*9MvW6nJUj@Mh}Z5B1r*(4mQ!?XO4JJ-6dd%AeIJamJUB$#JZ=kJCQY!NuW?< zeoCURI-0v{mX$V=;}Jy;G=9dPaAw-IyeYJ0!IzEyS;8(cE490_q^n5hptdPR9Uz!K z(2!4{5<#t5I&?@KOv@_Go5G@^O%oPVX~F{r3h4J6`omJx{>GAgfAcO)jbOTJ4#TQB z8~D%VeKMs&vKb+>)G}kDD5GU9l}R4DQZ-dv(RT4$d>ArQTR*hjRavsc4!W?=-~d@%Tf z+G5)viXZ)h6jZ^&C#I@x#8whGiGTHMNR&cgZDQU7s>y}FN5jI(=egQ`g{>ult)&WW zoy7{EJk)h$C?bg&<;;m>0XhG=yk;^3uo#U!!z$Z;ljP9Zd%z+s*dca5y?!zPmZILbV@*~{!29hBI7LJB{o7t3F~K3z(H_A1e(R~ z$y{cp^7Y~}yU4-+4F)m(Spi=)a%x(`%G6EbrII0s?_LHX=A~P{hTheCcs{;y%I>(|);>%fkw|jlR^D9Y7qfC4=QhL|w(57xTuf|G zc|glRy{n3PwXenkY?_a9n(_u3Ow#P_`D3H!GjfVTLHeiVHN)fob*E(*lH=Txoxq8q1{$!(f#U1iF9Q^0CcoD$j-su~AZYN%a0TxN$H)BqDCW%;G{;GK>R; zs41oaP;OXhY^0-mB=!riWhz)+{y}eh7Jy^ydaeo5Z^v;L^iV6RfI-W*I6o7C7w0%3gVSA1FDVHCk; z&14!!o7o-PhVcf}3N=hneVv|NKXu687zloDz=)ztwE%{nRyy!wFH41-@aFI5+9*H< z6D6rV+IGzr&qqJ;O@Qep^@mVfMd!A9qgt6{Hus<&H0@kp2{?tD0%a|y3`Rvyjueoi z4XwYD^zE=iZr7`u)u<9Ah!|E~Y4_|@}Sm$k?gY!jxu?mE`q?{?}sq!;kco`Fo*0_VPjeZ*9qgWXWh$M0 zqP5<@BzNST->aw*0K-su@c+=~hV7Qizk1iW?kh%RWD~+Ak_m6>tb~&yA*)qe6{vS$t z=Fk{`C*eV`z#t-=rT(a2Oc#H?Q8c1?FURs^*9wtDT24$6WT9mFsV1sy?*@pO<`}KV#K&bY$(@D1*jTwJ zd;TqP!O{V>yjJm4lL)tc{cibk&WI4((Vgy3By7e!W&<{Tx5bmNS5f*P*gI-+LDl8Q zl*gB^)|<^@Y<_TVAYE&FOhVG&UUsZcdZzCgfF&-Wy$S<@Nq30#&NEa|$UMkF7YwDI zc+UdSxmvxq9c38;jcSwnlSnXx$MHz(ecX7WB#lwHs2dmY>xSQ37n^c}-0CADk8y_z zD6S49!DumbI&iJHiF>^4eo2IL0#f>^#x$bIsP36VDH1Fgcer+Nrc!i+-+ol8G#*Pu zKU{=gdJb9ecKeC#j^nZeB4jLL==3_aLn71Kbju6wLW4%vl%+JVa`(Ol%YFIeheB!8 z@8Xq4(@=+N>-2mjm%r&zG@)_nVm7w&>aSrKFF5jF@*Ns~IT-ow;?u9b`i30cnG{64 zve@W^42*Er7w>WepMjc$JY604niFCV0$Co$A_*LsGPBi$!6XGUJ$>XkHqlBv9F6qa zP4ydv!(UR^{6c*{F|@`V*>{b13lZ#D@(wc)P!99n#V^i3c6a4%T~J1qlnc}2S!gA_ zG!D;Aqp0YowA6Mv1T$U6``RNP{JObxL1Nf0OHt$S#Rk`EHRRH;r!aYF=_T7-p6)v) zuv$R-la0p>ZHDQ4a*W_Pdg~8vDv~JX3zio@a;8Gaenx)e*7|T^S*5m!0;AUM8 zS(I_y;oHNqxVbi#!trO~52TjwWQ)Pm3dh9tx1um=IK*$%x9))p}Sw|k9dV{`37{% z?m#u^-w!iHLq^i@OvB+!MQWgc4_j%`;7`6D%-4?=I+>LyXb|sOc{?MfxiBkMBpjA- z1+Ca5B$-7~MJJ(d4%+DT;;zlFfnSP%EWH{@qYtGuiPgh`dx9!;(KNxq)A{^9%>qq*kMk*COJ~oK9v1Roi~XVqMFBtAg@zw@x}t^XH^26a;Z% zcodGfe)xyt(|d%`v7*&*tEqY30Io+UE0+p{RL`pCxvQ<#P~qcVnX82nxAbMq+BOpp z)TaJ@J&Z#EHRO$&<@C|9%-DKNhwfzYe|NI<+oDl#q0`a*GM1WPwi#^!&&CXij*Te_LMdyO?y$=; zBgj~({>ySI&0cL#pG5BA4wErc{gSY034U!!iqAhRyjtR_=Qks1jnFJ=6N=d8R2}47 zJ|BQ98g-d>oIk#4j?1UIEJ&gO7NV3`4ui!&Cxyo;8qOl+DFh#_~b*Q zagQ)?K}bGxe*zoCrcC5I9bx74=N-y|r-AOek8FLfC$iC*-+;)I=iEu3dR)oZM0c*4 z$g^5s9;^yhS%R=KLAq0?wAEwbgQ3SyPf{UU*2=of3|$6>I4~Xo)uS&SD`@VaeOt@F zp>s{0Y2;zP2$UGtpR+c68(=3OPkH~9B}PGtKFz;y2Y6KLH?CYtO)U0k6!9vwSH2bojAOmpEe=cO@|Cx5x8zGHrBZF(n zvrIs3bsh_g%#Maj*}Ae|4lMFblHWHKKcd3^ZqzLePL(xnS-e3l4=O~Fu*Lk9M!(SF z0`KXAqUX4>V+t))B`aeOvEpXt?IfRXRB3tpKTJ~0lftQ0HvNCjtnr=4vb>lH}O*X-x3mTnA_Tu0D316=JseCE|IGRKZs+zwf zHAM|dQU&<~dc%8P#w|ywwKvXZ+Ixuhl}0ON-x0bTD)+DL(l)OID7$tJ>Mbx2NcFC< zU%6+H^O}!hYGjzVxyP<<1@^U;XHSlyL^e3q37Xn_Ux4BxwP@l9<6U$uDb)CMI-{r( zr%Aeh1_mvRME-f)%L*6}?ex$WnA_x8a;<1V48qYpC zfgimv{|GA2iBcWm{zpHjp{u|Bvgqx+Uza;ifB-$$;1N+8Wm6H!+%1nipK`eO_j$kw zxEe2jDe@K_1k9%Hh{D?M zhQpREvm|J^xu)lbIXJFePF}^P$FvqQ~I_o(Qrn`|^ zrqmhmMF!(4$sAeNw+wm4;_}_(BY0-RdwFo9a(-`D9XSJ@4toguX-}UVRloIEUO~!6D&_tuV*+ zHdwW|Mf729BuBHNmC!KQc5Pfvqu_A$fmDX0HUL}3!R?HgKhpHWL%&o|R9YhTY6o>P zkn12)G@;n8f{`@WmHrYn^U+Wo%?(i8BG8jC0d6WaU>lN5lU9tpCKU3FBV2PXR~nor zAVE7ek@Pk$K_E)7?$FT|XfdSGvJBi5c7ikDE@bR|v5?33w}88C{cG4r2mV1n-^hcv zys?iB={|?Y2M%Wc9$Jt^Lq8wGvSUq4CNU>+R@jIYLv4K|gDV*g0}QfFw6W7&z1i5z zS6}jgH<^5+*YKMy0%Sj7_MEF{u5ug(l9=DlLi>?i>@Vq|Je_3VaJ<0sAC3*l%jk3a zH5C?>3kTV-Ed zdJ#rkR_8nFTqRzm{}uejiFOQ(Tbzs>D_H%DGNPAScO#wMA0r`c7xSS^=J;gfFoAcd zDM=w+*%yb~K@!=EM4#s=yln$Tu)w~ozaP+A?T{GKPuvvaH<2&e(m_1ZP-GJvTxYWB zto}*?FWxF`vFY#rMYKPT>={12OX>EQ7=M7Ix>Y)lC^i+EG}3H)yLAYASmP#G%WJPW z%eFjQlanRC$euSg9S~i1vsTwfO<6DBA!#%oh}fh{GA;i|pcbzc#0)AE$J2k zV>4A^_T3;|teaC*Mb*@Q$%v-DBtHhf^bpTQN*@QCDnyVs0xqXT)HAiFUD7)>R{qa> zrLxRH_q5a)P>FS{qihKRw!EO;4~#tn3Uf_I89z&2mE65Bwk%gwJBehSguZVpU?jz9 zA6t^po=Nh@GdTcdM&doYuwFK}Wp{W?%)0ptE8f0oUf1qN6*ec>A)I2DWzmMnesaHg zf-f@SLLq>0Sb9y$?l9z7RS*mKm$@WM#iXEKpV@C3{+jjN!G#ZF=VTed`B)$oBI7ik z&mXUjIKIIofn~BRg=pf#(FAh*irpkiQPfQ%fRE2r&EQi~iluj^4 z4qfb&u9)rsw!(T@H|63Xx4EDznTKbtkL67e3&y2+pyl->iE{Mg+tm5qt>=E{LRYj%@Fc zA+G7^kL`kb<>Cf?SMCWb+g)xcVUX{2K}MrXCw*9%pRVs{m)C78j%q$c7~;fmPvhx- z`*C!fpLrss=BauhTb;0otEsFUhe8I?O+Nk$#6y&oJub(Pr7C!|L=ftbm~^IDjP;otzg#~?w~R~8xb}0IeoU5%p3wnw$T%}g zHY+DxT?Bo61pa(ec7|5)w-{4n{9vqYK(2p3I^jc_lFXQ@VLS`%|Du5e8EG4%qFsi% z-Oq<~1Y<2~&NOS>A@`|}=kJ{nq=j~FJ5Qt5$BZM=7#vZ&Qup4C73X77_QF zXfG-xI|NZB-3X&BwCH?;dTVICg|{(e5tak^Y3P*u-GoqOwT4t%W++R`>~05{Z$-5K zT%;r$Q^+kB<$8>QEYlfq2nP{AqYj|g3;D;@4+reR=mKg)USMfKP4=KIO zf_Nmu-3b|qY{OalQ8xJFgTc%gx!{fX0^{yMzYAo-HmFu&$t#x>z->< z1eq=4m7Jpg*AwjeG357mzj1TLn0MHf`#;JA49+N9EHi^VD|DYD44&lhAZ_nfbUDkS+lqCn~{wOhQZcfkaq;eB3SHSIBz#}6TOLVn{C(f?l{`Oc_6dSl_2!stp92i)jhZ5h;;V1D8V#R$~ zD8<*%pG+EyRgDGI^Z3y!m`aUKZ6VQQ0MI$)X*0yqu~?{J55+!2#87 z#z`Ea{zJEMM|+*B+E}LZ_>9@vA{EV;!)9te+0rLHZTl(_5~HC^ntiWUYpDmhV%4S* zx-x~42#Y=7JW+6bXuhhZjIr~5Rl!=)v3+n+3%TGC~WdR z)%z#M32Ut(80G$$t*3Zg%Dq@{wEvu<3LkA}A{{h+Y7u0Z^mKz_A96BvOZl=wS$6{! zC>j2n$Kq>}r!`;cOvHpjH$J%MS6TU>czu_dJ+A8&j`hKWq0d9Gk!?6eI@(a z8Ux#<3)x(AlEPLAb=U6)X&V$9J%PI34vDj&b zG*k8%j(pD%nKD;QPbxh*lZ(9-@Okg} zPMnR;pg%!%(>4MRp>%kfh_=q@;v-5}1y@ts742S7mM;hY4e*`D)kw9N$;303Xa=4_ z$PWaK=u-Y|457YLp}KzpZ`Q2G z9FWnU1!Sy%409VOEAZ&Y9ukMgox{f&KCr-NJKD){EtAK$`ypp-)kbt-EW=wkgm{US zvuM$*7aP#~Ms#JNwd%g_);6e^3$(kQOWJX!0TB5@A63B8B&hcvo+;W{yyqE$mSbRu z0`u}uuHcf4vDmDNj0wn@4VJ2{K=`u45dTrGXuVu_t2j@+c%||1|6|)s9Y2R2OorG- z8mYg!f9H*VfWU#9R;`Z)cEw58jlID2r2C^$=HwHFzEiF}-$ka*07T_U`^WO9%L09h zwO0rJer{&ut{HS==Z(}Dr1EB`BOoK)v4gk_#B9?F*Qxl(WyC`bvxX9{gM!(MtY=nZ zF7(gG*6OSo=rO-zR1gy4qYV9ht2uQh{0xr?Qw;YmKlc#_s{;L;iN>3#jE;-{L^4^@ zxbNL5>ut6Xpqn-XU935&Ha2OX1Bgy5<0J64c0vkJZ<9y|`UT~mVJN3|wrFBJO6$(K z8k^T|@?T2$$=W&)MwlzbHMgaP=D0=YHP%jQg0t~3`ZB-E)*zkXG>dZcEzO-o1>J#? zNfs^sZ18Z$_<`lE>ALf@=2ztOIZ!H^daH{8-#7qOHmD2C~QZ? zBh3Whe5<&_>BpKMH}3a|M%!Q zpDYAr%ES2Uz;L7a=vZ`#BJ<5WpvFd(2cI6FWe*r`wnU@V`1Q5DH^Zlh4fM0gvZ77# z4*|~}de|=T9S@05e8P#FiC7Q7F3e=6hBY!f35~@|bAx6Rg%GHdv2- z97BUlwOIAlsxYTjU%{VHuhpjLganW*-B`V)4~rqi0D}lNnKZKfl#tTGGNtLj%mZc+ z7lzmQhL3aZ5~nyus9Wu(n;rcJ=-Z}p1gfXa`izlm&{DSxHg4NMg0Z{8C@9 zs#!AniLttujXQQa2Zv!1D(%P18P)T2eor&cn&8&uu}%UYVC+8zi@sFK2sE{=J=DGkG5VMjPKa`XegJ(Fs15s{96ET@!d0tiHDHiyHcaJL|zTo*S@ zLUa8Vf)v_K#qH`t_S%ZKp;3)-_h9M^8{Ut)q~xIFpDWRXdVH3#uOx=!>j@({3sdta zdm8ufi4Q+{2L;Y1+ah3LV)TAZJcM@A7PFiWO5N>F!hL^KJXhIvJAahYRNb*?I_)p+ z=jUHn!U?@@dLSd`T@rogvwQ&v-+7xq+KbwP};W8I`CLg6h ze_Y9ogi%hO?SZbHd#RF4g!X}4s&((c_Bsr!r@2F4ldJ1$W0aIRY5s+ORpxx6`h@OT zZXq^wSKM`@ABLj*Utt9c!`_c8u{h3)mLHIXY#s?*`be7ls~eNt)Fr4VS5!Em<0)23 zakdtvh+_nCk@^bNqs%l?SK(2ABL3O;*v&nwSgw27q)aumln+mlNmVM+R|IVV4+VWz zV0i`mCxR+AopVRM(f$^^MWOHhg{KNzUlK5DYVcwPl6{18{T`^Lg4LGYcxP5Pz6NXp z!_DR3;EbqWGb#bF8#m)`ZSzK5`ga0RqLo?Z=A)9*(ivnYPH7(U&4@*sO{a|D}YilBHLrxe~# zv)l$VKUjRn4wNljeuAy4XKRcwzs{MC0jq>e!w`)j4OE^AJbLqZz`)>A0^9{i9V2B( z@fO7&)5)8>iYIH)DK)3n2>VSBSwDurpynFVKvT7iq*;2#;5N$o>T5_xV~Vw!FFYCq zfmBw)BJrXSJ(o+}D+u})Re+9uQLH0}AU$l>HjN{tYxdFY>^mWq-%Un-{mQn9lea&l ztnfloQw{fIzN|ZC);HzA z2k6obfE_VljOtcB_N3JLT3sqXyQlytAQK#SBS?iXHFNkEes>A&>CqY9q6C8Z%K^5N zqi=4o$yIKl$hLOb1%bLQ5k)+VV+Q^UA~Vc;MA(Sd*=t#OI~{6z-l#B0`p3Q>KDx&| zsHN5xYB5QE=}`D01Z~=dcL>NBqqGU3T676j4FQh`Epxq+x&4&AO75uwZWGYX)S@uy zv7A7q_4(@J!D!$obwfHfLTFLdH08g6{Mi|l$Va+M>@-r>V1%-iq^u_J3%N8Ir*p>x zcRUi${;nePm5QyrRA+>z#ip!{HsOZ<++1wk1s>y+C49e=fvHz11^WNw+G_uwGq-asU6ulEQuDVBM=fq$>`-_A08tdDPEn+XDF(RpcGU^t(B<$*m#7CGQPZe~^ zIv&Z(WX?*w4vQC*y(r__==<9(sqGd~hl)&U9_I1si2Bs_D8n_uTqIGq_Ape2z#U#W zqn|ITSbf$1zR5KeG~Y|%=7UnYk_vmD+Q{vd#Hyw59QScA26bzI-dpA7@+B0 zwURVU%MdrAVFZ=>in2Ia;OZfhFUXWb6uO>egI8o zg8y5`;8B$~!<=>6gr)eyR{CQ|4XO|xJ#v8MNnp`|Sk7GTM%Isy06VK7bUn|w3XS(j zlykai(_p5+K4$RNJb9rZYt|xZ+v-5G%?w>-nVAFA6bvH1Szi$LhIorS(8l zsr@>v4!$s26Cov)EeVm|>P%4e7Kp2_$#yyHPRmBGEV64mE0u=hB*fzATKFokEzhas zlBCb!XIJ~kik!rOI6I9fbdytF!cI49OV+<}-?h{%fLO~CH(T@JcXbJgqb-FRl0pmu zuHcgTZ10cFhM~qJ{|A<3mQoG8;s?u6WZ2Ir#uoMPp)j$7p=&Uv` zQkyZh8CFPnyyfeYUlqlOSQ`2cD_kn8wfA_dx0{bwtLIzqH}v66c}T;2K{y9wBqd1) z+lMAn(CQK=(l@xm1D<#c9nHc)I1`=O5-}L0fn$}*ai(<#^@1J z&{ofS2^a><_1nrgc{U{~W-7<8a1y%E1YH*70oshNLDNQo@sSIzl}YK2hyzhPj!++&&po1_$hbs63}A6ln_E zb~>RiSaFz8VM3L|?VN>*}atV+Dkr|tcQAPMYP%ww(l-4OC;`Vyx;d&e6klh;GxfjOG z^j>?QlI;lmrvTg9wbLtGegd|oLSU!9oQ~!}$X#zjK(J&o1wD7t)f#?`^l!*4#3PC` zJfiCA?v@9ZJ4sEkAvH&$Qbp*rk1(a zsDL-Ivc*JKJ%XLhXN>}a^c9|)K8|m6X6j$l=mb9X81IBZVRfPfh$S<7!1W+{Xo`PPl{ zjDPrr5>wq~Wn(w_G8X7CP&*(oDg;yh-9QCTlv_({sKpo>~TTs?e(04`F2qG zs*+Z2cf4@Hefl~RRMDO>&VA-Q!S@md*ZDTSY&}9~^*A_S_5620R?}_>GwN-xdj0vZ zoMS!OyZxddYSrP&GEv9&rnIu{{@$9@>$3U$Jo#TOAY^^8(`t=s^?GvIs_kI!DvH?q z;=$PKb^5IHWiAyj+N*GW{d_W&&+aH*MdxXcu#*2VRfqTXZ!d-At?}a`|9z;d>GMon zr{`tJI=cq+j#RkwU7d2D@!iLrg-5LnB6F5rie(v_mNxBRL`5SiF^kkJvg+c1!C;dY_Q||`icZ9Rm&FlF#GS{=GLg^yvaA-# zVo{$!vv^o80pDAKjVmzb#@fWv*{3irkjOiE!71{CN|cl8l}(SY`xEavonOiTa^PQ3 z)`d(suM`q`wLYI!9UkAP2dN&8a4OGuc5PZ5)m@1?-gx>G=_^85)2Y4xMMJZ0!FG) z2-GZ*%OIJk`u>8N(dm^06NfokE05F|z~UV8rM6l>GJ?;|0emo%EQROVY7VI|vEZ(f zIX8<}iWC{|Jiq@;)NVmyOxpT)YyI^RcR@);)9qmX_2%vQQj)Rlb%}NpIaTzQvF-eL z#JHm1Rp+_y&0$iQD#QD<_`EU|mqh0w4m#(phBm|Fd7etg`y`jNbvu`1{S^Z4^g3(5 zDf|js;b|4PH&~JzPq5F1KMfokY54cpP(_bv!wFkG{xQKisco zKK1giwiPY3c{Oc3Bc8UL)ec0FdhZ#Nx9 zcol5~jn> zOz%@u`{fYK#Ft6Zw?0Ft?QK?XcY5w5-s_I^&sjUI%ca{hpLCOV?(Dw~;^8N*5_lN) zY!#U^Id9vFLk>}!tjO2%fJha|II(^3)lJ1ljt>*ez+%CDQra|8a?;;yRq4x4BHz@w z<2lWce#ftu!!*~0*uIXpRM6~Cf9Tz)Swi6kH~1UeGEQ++?*HflZJtg=G5DF>Lwi5Tz<8D=N(OTQ9z8y0-S_Qs&-@-?;3 zHeh{j7K=}$hv4;Cq7GklZ;BXD)Yz)t&!geQ$m-LDM8e@>OcduQjvTe>|F~wA;9>mL z6uIZNJ5`eBEQ+! znNvZcvu*W8=8y3zidyr3^vPrLXK*kWqtE!%_2ogs1riB{`eNa({)QH4U__=5nWzDS zdYgt=n~;pJHj86@_8#9TXz9W=-3?a$6o)yA2#rxw!ZS0^8OP1&H*uW_efcgO4!5BG zN=f;VDAGO~kfS;KvapMTY6AJ@Tq2Dj>NYMjRzc1l$Q4Wgl?5A!;MC>kA$j z@7#Ev)3I*8i3ATsy{)c4KRK^?nO{Xk@jsr=zXrYS-;g%%PU>X3*?1oqCa&AB!=S$I zw03%&i>7)%pFlg?)_cFaH61}GPd`(w_py6eti88*{&cC9>bW_(>Fc3yuQ6hxJY z@V=bZS!lDqZ|!__K5Mf-L>}(#d_L%WCVef0;?ho3#~XcJ{`$dQevdK9{Zys*P3_WP z?F7$77wPLzZq)N(exjRo`6COj*M%_Q%c}KTrgg^y*Pg-E+|HV;=V^B4+u!>>s*0A! zmhKp96n>qT+D?zFF^;zf4R7y~fozTX|!x`OoJv_=Q5J3>i zV=@md*in8{0BUoNY!QzH1sOGn1f4zuMMrE}qo+N-l1#NG=_s7wMk&xMWh<;# zWGi`023JL}HO}6D7Sn6>cCn*W)}1ZovnP07p&Byz$@77l{+y2DG8OxvcCFR$BVN%w z9|ApiBWKWbZu+#4`>#Uz5U>8hI3g>{PV)uh{)F#?Oecm*q2ukMuqdPPOpZ|fXz;~m zJy+*-V67vwOp~ZZbzIc(DY03)a0Y+tAJ_E8TAmHO1Qw43R1b#fM%1cnw4-%AjR82+ zp)RXY@z(y(Ro|kj*PdXXp1NxIKgO#Vuqc)#&@zVHv}9bGRJbkYm51Vy%9g|O#x=Ls z#X^$DzNn*2c1GCKN63qcw&$2T)X%Qe%JN@_ky@^7KwZ1jRk^d}tJ{40-MZa!I{)jY zH}^xzi*>#QNr#S2cL4v}!5#a3AfZ+BLBg`vb-Q;9^!{nSWSg(MU5y2KU&QT08Y%t> zJP7$Ot4AuD4g>ZxA08&W&WE1(-;SrF_+J+&rgdb~ny#Tx^3sC3?sBTT=J7o2qGwE;e_JJilQ+3Drq5p7T7%Wn_-uiQDO;w}}_<`vCe(>;Qn`qMnnpy|%@ z5s|cWd+qG?ii2wmiaHk;n5o!=&y@#IK;3=2j_7nfO{U_%>Pj3Cm2p@T#?_M8-%Zrm z*Rk2$OaBAzbS-*){vVpYGN=vjYnmFgxVyEu6WpP=Q>3`N6?dn&TXAcP6)#qt;ODrqz=U)pBS?+G=uES3c?a4upXcA<=SzxOQPNK0 z@}xaF$~8@FH7g_!=!LmS(UO*=U|zD3Laa;ei&9r3NawiyYPVfdcz$Xw(q;1cfe2o2 z64T6Yd=jljk*4tzwyfPgR~EyF(44tvE&IJVH*M!+&h@H*H{v{}*c1E3cZ7Qy7e$D4 zbm_HGqpIl;V|s3O;a;q1RWueR&HP*zg;Rg5ZkgBgVSax&vc>C^aPk`TBu~BGuW3E1 zX?+3sG?*l~7NLY2Ibkn%-3o z?}F8T4p+$1hvg5Mc0;7;UDb=!XzvWgUC)<8Z5!pudO0?`FOeaGxR*(L?{o@$*Y#SC zBp-Q9%Y^t2Tv7XqyR}BlgK{-iqN}!DC=99u&AY|se2duiaH; zA^g|#PU&qNyV%7g3x_J)XJt>P>t-}Am+%OK19`Z4L6m^q)o&A(<^E^z-Z1f}u3+`%(N^|OR%Twc zK;}3}(EtMUK-@&m7<}dY75TiyKjn(5l%<-#RHh6gQF2duto@&_=q?#txK!iI@|avn zY3dhfU4^s5MKlkg1jOJ{O;jOj)j7pkI>6UFj+i-9T$`P5S~Bq!i7N#O5e18B6N7{E zE%q8&94Q7T6?lc;O#$FiTqP+$H7(?IO&R?cx>PV-(wkSlxJ+WgC8g}fQm~aFkBqr8 zdY}02d{p|dSDN2&tXU`Fgi#EP?m5gwdvBA&g9nJsamO_x;@UH^BnTPCjET{M%m*kl zS5|v}8iXyxu#`}*eS)s-kzfvnERNg9q`APb!L1?E2fX0MpGu9Py-eLbDJi@$$u@Pd z@=QWYJ^2t);{D>Xx`h?FUAs?(=%Vb4LiO40G0wF6`#PR@1JO(ohIb5x`{tP(1J}I?9g8 z`SYLO-4591hd-G8Ka%mAf9>Xqlm;-2FwDEOP;8DCX!vNzCdbAleAPCfg2zAC=Xw0=O{(I%1%qsN|%ZD7{n z6>Y>9TGpH7vvF(Hi0gd03bR*ysPCJn`5V12MXwvwf3sTvur%nULn_HW#$)*9snc6DMI%{> zARw!j8*edw71p)4cnM zO@YsG8=2EzfeGl8_|sr+?WXH(8KQ|xDJ%5zD9%KD2yX^H4_K@wDN*{~^#or#KWCCT zcihixib;EH*oeUwVjV)Pi6zSf5(pejK91oaBFg7FXq<4Z3Z&;XBhp zd{N4hiGLGs#{^oEiP&AAfin<1uqoR7f~N1`3f%PwtE?5%f0m-iH1xW!OcSpG|Fcf-Ck(1ia2Ja4-MTyC!S1n^of3Vz zYwo)DW_s%4jtnZB5`REf4Me-VhzdsF4@6(gN7m_E^p(!s+VA+Mrpd3z5#U{OZl+pp zCVBJ5b4*$-uuE?i0l3^dCRX-RlhEk}t!`1eJf0(noHlAi;m2v>4|TA!)lF|W(T2~? zbyU!^Mb+hVYCr*4sNW5fHbwjpN{J-4^Kn;=(w(1jOh4X_S*boz%7WO?T~h!(!Lz`I zTWd1URP$1+EXAF^VuegvJ4!W@!L7YmRZ~HGPr@A6hI#!XSIfTd@0J1p>b?y~P^r{! zBdl+Wn=%QQ#re*{-mq!-gfUy>vzax;-3aF0>o;RURy@NEc+!7H*Vw~Z)UZdt!XlJj z$PT1ClT==W7|#(Z+XbP1ip1bOIQrPSvN974i&d8yDRVyXVufNo+mewX-x z@D=s@i39=76d!&GdTnL(m!Q^kQ%5u@AG#rwI_3MSlHcO41gzt%7V>8G{icyce!u$Y zKGF3Yy=ACtJklBzD5P~CrZz*06ONMkIUgfQ3e~`Q89(2Mp=hu{0gI>0zc2ICoPIlh0gjkl&SZ4hHwsD7Ol@DswGI8*am{uA`6)S7)_v57rii7&qp1a=G{_$7k5>B9`>sq)?K9Z&CQQDfaq4n{K z{;|FHF)jd4$K4Z{13Es3L}w~dUBru&kB9Ez@_3i{1bwUi(zvee^+Iy`_3A{@Gb9y$ z9(sOXkTjNMd(E+M+W#(hMroSS3{eeZe{h+_#Ol!?&Y&Lzd9bQZU{f-fzwO<}*uMoV z5G!OU4q^|<;*bX6QwKs7p_(HH3?dq+Yik1V7Pnp78h!3(3LTEpxZ0sfQ#0X zU|vh28|S~lmobk=dlUSsU6+?=O96L-7+s*zN{*JT-c6B5(@y9TmgwEXCNwU}p|*WH zyNq%*{^ToK(wFDk@wp`Da|BD{-@W~Xa?`|pA_~z|n$#Am82a?od2jmbY(O_|s5r(8 z4Syp#idl|;dxw9`voqBs9OGt($fVxa{|Z69KEZ;YH=i$n&)582&pPls6Zn1(#Hr(< zU`mdS(M8O}E9P$DS6%@$a2rKOgH72jiSTpbn0&w>O*t6FXJtB$r0`j<98A4(J z#F;4)mX+^7TIGNbRnD<^E?GhXl`?w{W+M`M3V zap?Og>u&)3)5!R@cvv+sJ7eT%y3pHbOBb=$CKue4iXC1$`;oWSMTqhLfh`t^??ci8 zKZ5>9nu4YgdslVph|AtK$Qk5WbLta6BgU=r0hqC_-yu;0hb}D+G+}S}VO6%j50F^d z{7)0bUT#}$un?RoxGfK&5J&~&#aN`o=#aMSF2y9oTd}5Z2e8xU5>uAOtanm70D)P7 zHco}0zyu!c2h-GtqBVY-8@*N{C}x5FVb1ig_F!SPkSRtkJp6(YT}ADkhWuUEQ%C)v zx&VOc^(a1Gsrox|E00UP*3nlurr*j=c$*I#w)J?kn5(}!UQ;)E!G)|-jK+!e{5~eH zg0uKAQocsiG?~`_BQ!L<;1%eZnnCvEE2IqS}Bj9it)}{-zB-6+#>j^;y2i&g0 zU6w=(pE8@*?*~Vg3Ijm=SWk}(xeO1w>}Q=1lm4LW2VrNR=f;hAg)6APMBw^Q2t4ip z7<{%4^X-oqN;LAgh!GZdh3^N~Hz7KIKj{7*bzl3=>4SI>SacIM7@PsOyInE~v?J9` zk86hC9aVCS#pTuo-Y3DYSvcUQ{_rO<&%=$i$vs2~sD-yxAc*73!!pYDo5)9Cx0*V7;2)AEFlo1?K{_|*y-Bxn zz! zwX5(W^uVJ~ao7nBaIJ%~kNJ$F+(yK;jfEFYrZ0bQTByN{ae%OlqjsD3Lhl=Q@gfQO zbKXcR#mAEEk+N0Ijm5jv2U2|NMHbqEF^?;reUWgUng%;M_8fBh zk)!w**4N>yIm`SkgmFY>t{{86K<2S22e**##>2J_qsG*(mlM)C=zVp^>VxmK@m+ zzZXA>6vn|nN`VwiVwrzhd%U!<^-4l(^ojjD6H~XDneXKlKe=JguO@sxtT-Ljz)Ux;(jhMTh^UUjs z%#YN(r@S`CMlVvFe;|3G`mLd$W0T)1k28|-ljVVwF@uY;Lm>%-43k2#^qkt&`Q6T) zl=cvKkM_ez)#OB<$3Ub)Xn8EnJ4GD4IQTH8;>Q5_x_ukc-%~=VeG4V&7eYBh$fA>s zQor?m0*!zO6VUg(P$K5S`?bv4hjY0gSf8^ve2v?=^Lh{gpbxQtmrbxGxnSt@esHFd z>mEV`IY1~T{(UM`53BGycnZ8JUUY9yNA;=lCp!>S0o#-jzxEs-V4#0Q=!VWPzjKDo zr)w$kOd#yZ|BS-Y2f^GSZg0R=A^h;HFmV5!Zy=O@s|_N~Tx;mIvX`4FaVfqLgiter zuNHvI9ou)#WzR;&nJPw5FkhgR3M-9q;;2c5u!ws-K|D0m=LTs&(7Sdp*A)a-zRH zE%#>7qlE3Li3NcJ`PTzhohhI1qiTVNu`IPg;QVI3THR8hvFk$Aq&U2nBj|pL%J&W& z1`lfeJgG_|Mpk55N6W4AZF~e&EpONm%uMn=xnDs_giNF5YRreoNHvw0MZ;GEm9tRo zvZ@RXHI8f{NqRPGI-jD}MHHnCXA$Z?;O2-b*QSdNYBiwht9i8iqt)P^fRn^vxUimO ztyh*AwPf;}9g8LXmhgyGI*;S#;(!>&vbnHvYCmpee$PW@KGyu=ltMD|1sA4gA%QRe z%2_AbA5JaC8bd&XzR8rzIk_|5(vCSNnY6V+eDGoFLlUD!;)R>&B;NK%)J*=NVYP_0 zeOe3s4JtvRA{f5KyC4%=yNku^J_{1FJn*ScXLcQHceU3IiH$>-T-2tHUyM$@T&0|> zW_N**fV0A$!@XDTP4$7XN^zon^9WI;v614~k5mMXfHdQ9({e0hA z{y=q?+OZw6DeAxXdC92&xCA%y9azW(KRs=Kk;{U?N6X2aVao`?qUbE>fvN!UTWbEI^yR>n>Hm7E;D#ZP?`&0e5Bn zqYOh8cCnFZ`gu=!zrBkQS0=$Vb)`7rdnGl|$toM|3&6Fh+;Y{^`#&y#5v*L(l_J2P zhq@%2*28P_b%57_H4Cz?NI#@TKp{J*R}{~kn~+fMx=;?viApEeGMilJ_R(L>xh7o< z)nfpVjn>x`M1*YNhqKEL1n_+#eqTen<=9lu-kbj2g?KPu10}8@ysznRw5yo9rRt@^ zU|tS0*(<{&`cJx_HtTpav+~m%IL&l(;6eaxkl*Yu0Ck!e^Y*Gy88eeRo$HI4v$)e+ z!w$8&HUfot+(7}RJWrCcK_oe{sE>S>yEwAMlx5f`DGaMZ{e!~zveCKfXexPjWxt~h zaHY0ET6kIlCn0;SJapSh`!IAufad96{Nt@lp_h3}#s2H2caZ3Ym!5$f=mch#m2Dge!L2)1q%B1$7!{IK~7!PM2r6}oiO`K z79Za2QG<8{*rFR;-{2gvMbBWpkjqrt^<0)c<k$|1IZ%RF(0oLCrKK?_9CJUguX=+ z3}Ja``3i!BzT?JH$4v3=-OMyIlt1eDs?3AXG&^6WoGi&Wnia*2yf1r0ass_D17hUFZ4XVHTi-d1Zj|nK9dvy>F}R%o<8lFR;(A$6`tRU$5)TS@^Wb&NP4N*r7 z{hw6+xmorp;(fuc_KQpFjlSY_hmd=lg0ACwmNK|ez(q{jDx7_+vbM`(;qy|!ZpNxf z@bz9o=H_krlu_VB($-2Q~uyymyW!U1gbZThXZ@gVjT zr6zv8%n|%7J71a8rVDvdRA)49{sTbf&W_r|H-LzG5>x+q_fG5Upo(_|WX0Hrf3;)3 zH;frVj@i*ggtwo;?Sfq2=9keMpr2?inN%0V54}cYo6rV z)neFTObJbPw@Ve}G}kd(=P76Y4+u)u()-3{qp>c32=Xg~W24hIzt|+K@!iFAJF+S` z9FhjJhKYOCWd`%IYq9-|{~%%Y>t@PpFhq`J0Zq?1F-Am(&s5y~v!mzT)UMod;;$c$ zzq3;iGc>k~gVPo?rXHlP=1g>XIJ!~_h4<(@KJy}?>o~^kI18|R z#ncL*iF#@FN!Y$(9?w|1@z%2Mvz!G)$O~6ViW2geL3LnE5{ZJHQS#lG+MZ>nWBb88 zd8iNme(sU7*)s9LsDw=t-@eP|y4ZK9{2m=Ul`f#VIc)WpTlp_lFf{1NFP~bgOvPTB z_B^+YGD9U9*xQliI0~P_Y7PDNK5yQ&!+-Y>nFJhyfmda@;J;*&;?Q}Ef7Hq=le;uH z9DV^0a(KA9xoH>o1RE6!1mOL~Fu3qshA97ToGQQXyyv}G@lW2TC>aLiSzSGn%bjRFG8gPmx(<^AhaV`>i+eT@zeJ0 zvt7fVtTfd!vXu@Vzb$BRz?0YVS;y1joSNu!*x7T@qqj{2m+;&~?hlm)qQVx-uuX)w z+~ny|4zYJa0h*1{6>#0O6#V4;3=E?|Tw<)-ZZNP7FtNWs!SZWU^x?`IeoyAiyFuu< z>@!l58Mp%pj==h3=(S?S(RnAFmnM0mVD(`4HbcDN=vH`k+Ohd2le^qmpsv%>DV!%c zj-Qmm@p~tE>&Aq^N&zHMgS33DpXcDwAphqr-QS-P2Gi!=`V?w{ztPx8-DkeBpppo) zVPNyR9>{$0k~nGhi#q1(E{fU;Lbu}s_Q$kC5%XiaTQ{}^G2BNp_wc8NVJW>kRveP3 zJvT6vxqpSA!C87&VR=V@@QqJuAfpc^+3qW*9j~j!GcE?78lFDosg2S}|IqG@Fqay| zj_Cw%?jV$<3+*A}Y(0#Da|M0>p41DD6H|4Tc61Zm=H?_CuwA^jVfG$3^JdlfBj}tW zD&K{2*aGdkreBXWVcYHC9Xkit*{Q|y54v^pX}`Imq8KI}@!tnHlUlqg%_~g;joeY< zy3&vNgg;wHFcO1j=GO!I_y`lko#+d+W02w_0(JDPf(2DLv^oqC=M7_mKk2X!NC`Xu z9g5q4;2s;k;IsQet(A`V0=|g%Y%~~cH?~sj$|m?(Jm?`e*yj03D>oA_x0o*=c$$QA z#`hVL?YhD`b`i%BbdsLweRom{K{y(Y@|!pGVRkaWxrwndC4@R#l4-nbV{H7-i}wa> za))4FrYW(4iP)A(i!@CHI&Vl9iG`@a2WaWff`1-fmpv5Af)qxL#puH4RPw>Cd#xIW zI-r}hqEH6Kd$@*HXsX)2G#xL2K$3_nTl|!_ic>MyU3QtZpVkDXK6v+Lt>|c2EYj$H z#Bt70L{_ArEp=o^RF6ocq$z4ogtf*E_th8Cq6;4n^r!T~@}qi3Eem$M+!mspEN499 zt8ZSGjdS14DKWN<1|lsK16I!p}hj;VH0q z=Gi1Pu0d(p?rh((t4#U)^YTmGdiW=w58hHh!~{NQRj`t5+%P)I3%d><4CS|KSzIklclS`4wnUC}s% z5y>24{wGG#yMu6__@MCnj#1a{pfa~;Ck&@tZOV^sC@S`p!!g3{g7TH_6CSzE3})iq zkb&0s5!s=AIF}LyZnike#^(WEV=i&N;HH*aR>zR6S3b^5`g?0BU4bZ7!HkOoRZc!V zS8>B(sv{Z!844ZKBY|bO{c1!OPOz5A^g2_zDe@&Yy{iB+~NS87`f$dC>EMaKs6WYaCD?toKVO~ zLS_m;(d^hb0V2FowRxd%#wU`@vtV#8(D#n0dGl$61HNLiflx7xgRj%!ux9uWj1Lh@ zJZYD$@l{{xWpo>^ON3H6&-R+fUuDe|Zwy%_=%5N?Sdj(GKKPGqTV%B(6EPoO?^gf#?~vXWk+Ane%uWFb)pL z(NfO$0%rwOajbz20@)OTYxxE(6iQ$(=rj9f_k8ccHXs2D$++~vVf8o88(l|5sPaL5 z<&thh zn}3m#%~y{vRh?q~h5d-rWyTTJucZ^>_cB>Uvbr&*0y=N|X_5O~E#lY74#k`EunZaa zHX)!wJQ}X3IhMUvplY6U`rmO4zy^LQmxSbweAc!bZoP{@9T6&WtG{f^%sQS1>uP2h z4gVh(AeHa5V=}o$s_Mv*P4QV!hgDAuw{_+x-Lwkx*dIhJLoYA*mC9sU(U?E(7Z)6& z=H(w6OPZL+k_|iPgKIk>JcFE&ec~36_}wlQLBy2t+c5Q?1pNDXK2db!#8-}3?qRDx z6i%u|r;Qmi@Q#Hi{;JIt?O4V+XM!Iyg=rRzaF8PRI%`Uu-Y~1Y zm0T2g|I0X{qv(sqYcZNt=a=M)U&q4s*S*pG$GcXvdG?>F?pom=Wg5IA-#t-rSegb) z-=Ls4bgi7E>H-ub#wiB9rr&Pm#NUEO7TugNPTEJAm3G7KA{5iFzaZ8-bU24A=i}(~ zYO^^ImOk;01m1pAzW?}Y2_uthqrmZ1;L%j;%MVEyd6{Zi%-W=%87h7Athh!H-ab#p z^Q@!0F;?b8EWLhBqR-qVIkKSHy#7{oWkHpS7=U!Dqsw!7|Fo|fnX_{ysjtX@Bl!hy zO%DFP($7V;&u{_^Y2_USmhleeL9>4=2=+)&i>;zXtQQqn>3ekR*j0-i(({(yeIkv& z?x!XuHU&QVKB3(-g;oS{&i@kf_v0!^?7m%pp+#=NfJ_*S*w?6c)GQ2AX7|L#+xmSl zy7l;3Zg7N*7oos|hb72G!!eh~ln5+vs|YuXl6%VA^l;xx1m&j21LQf^%yG;itpOM_ZM_vqqSM6?^`9j!FTN)iFUU46W z#Qln%Ka0wDC{K-71ZUD+n6w%)_m>yIgK3Wc*R=}g7L=v%q4xYjdy7#F2?FBh7| zO3HYS2VZ)73};p8anf-$DT}CH#a575Hn&=S++`|u|D~;H+N)LU9k)Zrau80kuTPmq zPi`|;YeX49+6J{KMJ}~&%|JaH9Y>tSf-UjqPcsgOw;OIeJJ&i&95P=>$2BB4{|r7Z z9&)WXYfBS{^w6OblMB1=%##pT3w~rg?zu6p3Bly$CYxLHmQDhQH*@7N3)KOK*1t6w8F>aKzmluhr6zWI%I&)J z%vJF*C{I_c@+8U!Q@yWF`C6c_vMa`w9c}wBN7vp$n~J;M%y{EmljcLHZLNg_zZb|V z4`3g?t6mF2%n3%{wQRYLT%#RcwkjnSHh%Q_fanBPJ(*wpa0BA9*jslDjrY3pG0kmB zyjMhXu;AfoOZ$x%J0ySV|Lt&ylB#jFMv%>E2o>TgEbJ78;vfEMMFU8(L>mKUC4}4) zII&$f%YPU$0&TjBrnq8=8pXa40T!)kFvo~%Ed24|uS7>g0G=oZ z;T1EoHU=aVr3{A3J@__npL%3l#ND{#4h!2>)Xlgon|v%IFA^uHuQYDl=wGe zUQr6T`-Qe{*%eQlF9HXb5X`Yn?Z@7Ds1|W$O(!Dq=@I zCAV4&Z{NrHmRdt88$xLO2R+|p904GZzNi5nm?med-O}AZicSBw^y%!?kMCYs*Z8RF zxk%ADZ|3!piQ_Z)o-)XlY$bL(=R;p$jGa! zK&e`g`s2oFMfrzQ;H9#Z(r4jVe`UN^B+kLg$eB>S7(5+p6p2zdykA`$uGo0Xw}5Jq z$Z-YF$H!-^(!|f)+7{dWFWXB#M2(n+K#q|@Q<3j6HWKajPEM|UZJQQ_tCCTtoA>vNQG09H<(C9{B$M!0)G`2u! zGcwjZtN$5)UoSBwE=VnjmbyRsQQa<8N{sBG6B4R zvEB**Zwc}|8=dzzD@Aoq21d4RgOj`isK*IQ?bLM0Rc9$Qy}oM?eyGps-9E|lTT#a) z`8HKwq4L?r({3LTqG}D5O|gMJ;(g#?6W~y;$}fuk4U$yGHKQBv6or25aIQbp*Udxg zW_V-F8uNY`g(3~Hnuz`b^M{SO5t-p$wE;`tKVQXyLoW_yD{-u?6^~U@ga|)Va&eEX zo^qmx@I>l~?2UuwtJK(K@GOX^^e4uwa8nbdY}5%{7`Lv z&J+Y6tp9@sB%}*lMp1e%@8>3)nW|iiacf#D3f+Q5&}RtQ)|6R3#$7ZDp|1s^w@-AK z8ZgLLn8)#;v?1%NZfvyJkB8QpRNi5)-$WIS3p)J$K_9k%tKZv`u}{2*!oL6cSo-sZpr{rXIlH@B8C7Q`)9yAg+Qu?yPoD4T#kZWDN6()}c z>bl{r4te^EFIR5=h&cFUANls-VqXH9^mGlw!xQG0V8ga1l z(YnjxB2|5}Zd=ZO*-iMONYt0hzY`%LpY(=w5AGz4Dz`hVKM0|p@O{U`%n+{)?kPq7 z-H}cG^aWM$56h`=}yu-qL3O3juW0t44B6?{N}N z?K&b`2WwX9Wk}I$3&nu1^`Dy;@>5-|5z!5L6O=hQS^5)}Cr96Q+N`e;5iN{rzT}bp zX>y*57eEYr@(q{+e;iwyb7AJk#SDBtZv|7xCtY zlJ856TylVzNps!TNNGv;r!aMktJ>z9(tAO#B^%59%;lPiaDZ$HrYxBQhYRI<(E%Hr zw<3w;PWA$4i>}lO(pd5$)$({@GW|TpiSe(U2F8s$()COdX3KG=5+*OG0DSt^)j^vn z{+eb%U0m&(FA%diAc{1t6G7e^!>}}BE!?o#2Vr%L_9yh7J%V2(rj)++!g!ExHrwZ3 zy9#D$yfKcWmWGS9f>JwswPOsfELiv^ujCz}%4LFbDhnp1bvQl>w1ldhdi`d2Vv4dU zS@&t4sL=ON_gF-Vxx=Zl{$K_3_z9V0zv-@bNkFM{nR#4=wYzuk zgo%eR%~yJP_4Bh8C4QMZgeFZ!2sN;^Tzo7AkLO>Z*Tt@Sbl#&^eBMQA|!g|#Cgkt{uBm7s_Om>fD<&y7E zV~a>LTR6ylAG=jna^TRoBjVd%oU2Hos(A>_JQ*7fzey&U-Sq(l)iyNTqA6R)fse(cm zh{=3=p3o~e3G1yNIVqx$v&Y!7?e-nH`06S8qhgM4qTM`$VM*!Lc?7dm2fB4Wu~R90KvFkD9yL8DTmBZPf3h(4Q4a%a?x-i?IU~fD>YoP7pBS z=(J%mbb5uQZj>3#>c3(wcpEOwH`3(qDL1~nNP}ePW5%gZit7G^?+{XWhe$~RGVtHHspS9s;syB|)ZGf%#~7f(AF%bsN~vEfgca zFjV75Q67xnXUMM5P?UBMLFXZBd6qTM=X7%@gv zbCKIN4FKBH+C1LZcT8s>O&fWTqh+ZrpK=pt%hS-jC4`5wXZoR_KL*=np`hW&=dqP_ z=jJD*M2DtoJlu+&6i`;L+MhAQNzF$Wvl5!gMp*fOT)=>xeqntay)Sc;V;H+^YL11> zkce-)#<#DRFj|q$h;-}3rVk-Aj$QbW717LaU>X$WQrpj5$@;%teRRq0+rS@^Rz&0X z*oThwnXe^x=tOVGjy%ZV8uaLhoQOQ+4`X^_usTUQm??n zF{DX;<}=be(fG3zH|5thQ=}0}&Psi@_pw+$T}u%5&e)&0v!O@_w*utpJ)GC!s1*B*_Br# zGSrWf{ZdPGM?n!8+*GX_$tf4dk!oR16aL$zd3+?goM%5${8CM&M9 zZ4xK2sevvDH!m-`u}QIT?1Ov>>jHb2kGmfRziaTvab^!xnj;C zM@0;~^sIx6ovf)*(%{r<3Peccbz>y1&QC@|UZ(NPfX;%&5Rk5~n>s@^C$xe%3P240 zQG6hww(Dx0)8mO-e;Qv%&4V>*xD@#d6Z1N@*Uvn{4)7n&FbvCvJa;eXYV-w`edsVa z|K!t2C&XH+nDX_DOmT0$y+%Q0$r*|(Qfl@WQ8qdHm)(PN`|*{EY)a*_$Um}6a2fVm zkTtD1Vscih*=SY&fep_nv$cYwPxu8+>M69dv~^}pzcksGkH&2VP5+F+Y<1&l*96L; z0k|HSEfNvsVJMBY83a?xGrm7jqi{|NW>irl%xa`gi}6j-4GS}>T3*}TzO5C`Bc^GE z96e>n)@igaEBTuphXY(XW9E7xXb&Zw7pD*eWp8%Nj+etqYGxYM$!%;ZN_SpG99Q)! z6zq5AWKY7vMR@TV73)w0iktz!wo;QeMxs(m8>Fr8V$>cZE|4CyHSgLa8V$+;iPFuw zIp=VSX#20N6=VgIpH)j{wuV1=auHjh{k8v`VElW+*+}+1H%6ADv^CR)U_5=wo0PG{ zKUob-Ev#rSFhw&n$$Q^MNb^26@Y}I+jo4Gfx~u=psg>NV8MdI2kfkdB{56@J@5jge zRa&*+h?5Y+_6+mae{*oK!EeIK0C$1doTR?AAGM@a@Q5?VH9;MZ>U>}C5Ru1eM$OW> zJ~Qg?aJTzu#sx0krhw*tO&=db?;x8W=d~uv4Pt22NfwUA@zPXA(>p|`&Pe=LHMa$aYJ9S-DzwOx&E|ln_vA-_RX?Df*bWRdPa`2l9w8yK*axV;}7oe%2Nbt`rOu4 zQjB~akAM>LeLof{$6?(jc1%LhjT2?r;OUSM#iV#fiy+aar2rp)Z$NGLDf>_3IWzvZ z86mj}_t~cRc6VqSE^!%P2`yR#}-X>KAFD2R~zT44NgurT2Gko=gA2W?oUn6T^<6UwXqG!hl z+Q@?<LB6f57u@v;abQ83a^zDpzdk5!R?VFSOF#wRM&`exI+neEx~$1!Jx zJo&26b|U-H%!e`SoD`rZ@8X;~eP5Tf;e7u%-s1{05=$WE#W+j$kj@kz z<);1`#ZUiKugV;w57f~4oTW_|hVdD(uP*_Gkeyo*^r;f%SULM`=C>?|wc*8fm6l~v zV)yr|RyG*{{x3i76CdS3+Qil0WpUlo%3eE#N=dS?)@XGxi6*Fg{^;geT=89cSouJ4 zy!%#)EhF;7v5LW6_R4avJTv|BUGjI`in^KvHsURqKzCrSk@2PA~zq7}s12!-_Kz@FXqTmONJXr3rC|DS%jKWmUEX?p7*-cm1P z-q3i9ehR7AkqP8P+>`4j?`m=p{Hv`--YF-qQ^NKlH zp5q?BI6eZ*H8fYIK@&@iwfRDKe=pO~C0e|cuA)xx2iW|OC9v<8rQsTkCFh8pnrUY0 z6reiBu|5mwOPTc7Ol^6Tus;GlTC_*CiSqZ}^P@sU^RZ8#xwFH@+5Jd{dGu3CEQ5ZQ zmkA`xjkWBnZ;|Glep#0xk+YBM0DA2OIw%OD zkPmN0%s@}cyob>sRN+UjVq4*E&g6Bz~^ zy|?z^5!o-7e0~B7mAN6rcp9tgA4#PIDtfamkfYFFFL}z`ymiI?<>?h zyQCafH#?9+_p)0iM2LZ_EA@j)Z|E=X^tJ-*sl;AU*YwY_|HHrUb{(H2BYc{r0yp%% zj3(nhBlnTHUcNj28dMZta^GA~Q_dtEgyo$!Y;@J{^=+k2g)4t$|5ShZ%9fjUQcH11 zexT})z-RgtluB{-EbZEzb)vB*+(u%xF&~tk_UUJC1D?r@^>WIP(VrG0^(S19E$=|? zas5#TF4nT{^xpv6!xZ?UWT)7G$lR*3dL)Ea0~KtBU=adD3W6E(>L10_WA78~@OSQaw(q z4$;qFp?pRZ? z(&P=p=VdEg^sVA4)=n2|B9#Z7Bxn?YI{Z+jvN#=3IlrX)g?;d3<`v#Md}aBhRnXWw zUTID~XqwlqHj|P+yffa|+jfLT!Vp0)_hI7NA>A$>4TmDqi|i8$Vo8*sY7A8L+(m>oci{#0dD9Ykx_S|A> z*0rHXix(|WthhsPr?@-C-L1H5ae_;M;8qCkuEm18yK8VQ1==s?z4zSj-yZq9$JkkG zt~uv}V*<5=;KsBh_glD{q;&n#nSoV54%3HGNOODSe!&e5AHB;rXXc7GZyLwpZ%%m* z96U!;ZP&6ncujvK4)by7ZrV0UNb$>C#kGdg7mbP4b$4>SsLVeQULIFe3^IL)C!?HM zfMdBq6*jXm12+0aEHY7!s~zaAl54@eE0|VUs4w=G5H()8>@W-eU{{L@65{b<4qWgN zl4%?5A8-Fj#zBkcDOZ{M5sriiE*?FV>vvn&9R+yIJ3nLb#`p8(AKeCUXdS!dF+G_^ z_BCpPbH|S>N0=P>T6LeUj_%)FMy&2IVE`s(f3l9OXY6D%wQWM3oOlsQziL z{>MYH{Q4Q(^)!Sb#CtKh9P+f!`I*bJ(8!H*EBcAdZVB$d6Y?BQ%z1qhF1ZPL zG817N0cfmI^zZAnI>%{+?w~0q+iK=wD;^;RzbZAnvf>NtSN{gw2hblOGVUiX^ER5S z?3N8^J~(7&!i2X`R?>%dQf||e&!$teTG}1)K?!M)qDTW|I~jB+FkC8mCqm7Iny6_R zJ8Jvm0!sWCw!4=X=L)d52e&-XrT15k-MEm<@PX7ly)Blz>TSwhCS9z4d4M7lcQGFyJD?7p0k?z~&NP%=O6+ za+Q%6Zb*4$d9IKVg*QcNE?cw?8Tk)9oIQ5;*<1tBw!pk8Vs!nz!*yvp*>z(F z`h?F$M|$^r(C5z&Yj{AlF8ytS0ttgBt;R}ci)_iSny!D0JgjKNv<*_KmCmG@qJHYoL9XcKGzB>4LlOa}^N5ou$h4lD~ybBo+J)VCYS2okrj%)yk1U zQW5a=#?uFJlCyq!2tN+t6Tay?2X#C99i*}Zjn2mvC)IV4cc4+h*5zV@&L(rlB|04g z+ASC|w^hbakzmSBN72>G6C?z|iG2^wfMVoiE@5vL)sS}x^=(%cy$bvl;r9F+63{$7 z#_B63#%DK3+3)g$z~wV6MKIolBD0eIb8yU`K>G|S96CF|x4;y}eiVD7{$(VWmMu}R z!=xh$Ntk8$Vtuo%Wo%7C%Q_~^O7a5&!)zdSoE5|>`&4n@%9Jv704VvRT&VG=UVtnp ze$N)2xdpw2GB>erzgr{tl}9QFJnI4jvb zut%$XU`?5pk#*;FI@S36sh9!TUhl!#k)Bzbd|#lPo@~g=uqNLKa^54mY0M?lnze2e zYk{Ibn-!Or7`@D_OXrl(O*$g(*C2Ip4VCHrj^tHUHxUlFRW_XtbC&_=aV$4keRA=0 zf|MWXCm>q`R!J9ITN#B~qqYqB5S(Iby{m%;Ae!6W`Tv*&r6qh!x)`Fw?^v}DLblu# zxLN7?owemPnkOe3BpSAYH8-6rvYVNrD|2c|oBi#==A~|CakSjf%3M%?)@At%RcMZ5 zU93PUv+D`G%9dre9{`5UoB=%t{hW~;EzpR8KUyV(RyRCKqU&2KkM zUtH49QBH&}j&+(wpT+;Xg)|rr&%?Wr!aE>2%9V0$HglXq-&CB>?ApFPcu~~u5jUGO zXNI{gmiA%msd{(Q;*Ugz(tLqnJ*ng@Kr0V9Sim*xTbXm&7xt%ZkgHphbV6@ZRF4Y{ z_xE%+pbT0-YFjDsCN+qFuFC)%kx8Xhf`7vgJFifL-Y0h))J##`WUBD_BrLYG$1z!B zZ0WVChf!C@Ye)5Q$IdOQQ|?^i)4djJj?`B5^qVeyVv`a7WLC91pBc9D2Ay5h^lkt` z?Qfi2vrh@Q@J|Ri<&kT3N7Ejn-PwPm(jqHVF;c7TAF5uvkkJy`TS>sDFK3D^dMDRf zYp7~q&f#64{N>2{JM6fU95`zG*d6=np_g8|Ae;_*>w#LWvQFPvB9^^ES$EYzkF1W{ zZ~F6mn3!t0?1X|4VM#)10?NR~eh`IiD5}9nb~M~6;AkwvL zZ{B(pHUEfbvwqY4Q-XzH^W6LmP)Cd>X0e?-M)NFrrgEUAWRjDpuhazuPEQ(|Q?%E- zyhO_WitHD?o%irnwHrZ-1p=4*_=)7WhAsq-#CjDTQu`H;kb7F-Y6U)CO zH4&^9kX3|>xYEEv(#)FUEr@mVqzM4{6)>@9pHL`jKM`%)q6hYPc1i$cMKY1BX-4E? zijEQe?3e^Twhko5eoE7~#n!_3XEMP}Ks_R7V^WxdzQnS+jWu`B6t z)<@fP$0~8Kz@`&^!ddF@;(rYMt%m5f$F_e|V(gkDS$2fN*yVb^GBsG3$+{VQS(m z`mHqmP5>4a*1uxA2aXLocMkeZej$EmGqC{@4gU66#(zLE#QfHQ4a1fo1(`PgHw@uooHXg0(fW$wtE6kWu{riL z8kOtR{3BDX8vD^z4Cjo#xjYeJ4i^4|ABGGR6`pyvvR_OPK_?U>`p=%|%0j7SY;9o zpFodTUeVQPPV2yn4mBp)DN;d7y$X>H?dN}d8irZ#=aTR+x#uKzY3I}f0>zN~m+A}l zcoHj=ch#-y=sXLY6&Ehl2817MMufc`x?p%S{I&=+O%7V16vjVO2NR8H-L02S&!*Xz zHlobLFcc)#7eo?i)LLiN;Hst2RlZ=Tkxw>|wBz7(9RQ^76%BFxoNdZQ8;{d0&1b^# zsgHhr(xpgba8I_-1!mJEk+9;D$@(rtFe828QnrK5hCJ^+0jp9It9cM{fzkUlFlW@Vru`?yrmMh2Jkga`Ka z?$i(eO&hV(<>Hin$~<15?x;0}s0s&GOX>LR_es-|LS0eP2nH3JM-;T5_)wP5iu>ip zrf_`j@Dnp}^fIZU&@1#8#n|7+5hlyvWZNAXlKi;lD#u}J>^Q|0Kt!-7p~!5Jsdy4X z3q?rL%$7tfjSVsRFnqCU_13zi*QBe3IQx_g)x7re+qQ z@R~McY%0bl*1ff(ojr3QcxKD~MY=o#cv4lL{QEL4Wx9BP60*7XM&L|g0AbuV!}f#T zDCf&|7o^>Nv7Pnpz*4az#vPZ^cIDFfeTMOoiy!F-9LvXWSz7WRM{`VZJI|V2(P|>` z48q*nSGttfVF$UCo*5^Yz^@h}9JiP}?asM^=&FvQ6U+i9l~(>B!Es|N`=;M^K0|L^ z*OP}Z+23w>Y`TQ6BUVBuecesk!}o-!@3X9ckahRr-*Mf zmHN1#LH(WXTQxzyU03=NhR{>&w0{;Q?dN-WNGYGP!%GKzwO!c)EB{-iwl+Pql&rxz(XawBJ1mH9o z?GhgYAu01?cL5-sOStV^6f>*H*6sXtGOlj?hsps@Zg>B5*EZc*%ougcS!%-{Mx1^5O2@EU=`HO}atc2T*g zecj*z%JPL+b8G7!E+16)ucs^-8xDm)mEwY-GI27HqC>urH7IA#ByRjUcZJLf4kwFQ zi3u~m(3JlX_jFUsZLSUh)0NH6GYd0iL;{_~jU8^1f2W<@XR}zSPzo*V?|+A*dFWd7 z)2~1^s{*U^;QkE1`>N!%G0M13I*1?5OJJsepG&X4N(QqdyZl4yX!gwL`;gc|^n!gd zj*kUK5S(`7a^x{SeHNmL70r+bksM%#1B?D!)}sFJ81gkcFNsxD1W_Jrf4MSmbyBt{ z&2z9q0i10xvoSsv255~Jou#Hq*S5AJ4S0Je zRKTVV!Hyx61XgroOZ@-AIjx62Fls8B(LcE>``pAcc@D@?9}e_xw~=`)!hd(r*_`S4 zm|di>{(8M`vARcYM^>Hzvv1&{5i3Y@&AV4*wGvgY6uKd$_O~gXs)yg2nVV{iEAxV*E97qXiO~qt=@{z}V>zA>yG|Ir5 z5EzCjBe)S>@>Qqc(h`&HazSuzW{))8I%5&{W3OwzjEkXg$l&^YwPVei>3Qnxa*iP8 z;`XOzL6wyz-f*u}F7m^*x?xDQmj4`JdO$ixrDyAAgNW{VSO zMOp@3a4{`Y%}bx7j*^%9`pr#zT-&9~+tYFP$}ES2x-yZuDbc2UscKTAK z&k9a%1{Ag!QS$W8!t-3>pi}L{VHr~CuzMxnH*e;?GdRj9sPX~ZlQYS}GE$Cf7aKIu ztDVdEcsGuv_sQ{(*0?wSxR=i*aSJvVzGWnrsuFRFV6@k9);PMuD6~T<36WdmrNPQ_ z$NyIgsP2&{i?pk^*TeZIO!BqQNwD<~l3om09OFz@(|!&wqDt4TnzG5rgLtO-PyEvQ zJT!RNhs*rkWwk&_eLuB9l080>GRzuGP>r8d>X_Lj@zk*p?%@DxUXcY&Xn#W{l-+zp znvChCBQPw}V?RwNd81;kaQ5?SIxA>^BAWbwL0nOrOHSx8mZ1+w;y1E@NmUb3%^fi) z5@=%(hBz)^EmUFo>Jb*&HGW|aRIwMh8NYP#?9otOr)A1q)|#+aBv>O&0iaCXfP|#O zc50*j<~MwX*gXJ5u~Oaz(s2vayUpEX<|NnmPcE3yhxQqC2s^wXydmjSS7=N~jv$hic&7qOAlZlOrlZEY6-JW9GqHvEFdh3e4bkq!;R0C*jr{mUQRv z056RYm!?^74mXdOl||$BhgCenduu6NXJ*V>i9Do0dxVUETs%*xfH(@54Ln8 zu6!RE_i-^1*cI3gzZrN7RhhSvkk-aNi%-}Tk`40UUeu+ zg0ZNONYQG_YQt`npp(fZgP-!{aWTLmuN?-ni1$#V4lAt~cBMTmrZ)Q8_IBjx*QrRJ zaGIz>XQbQ*mqYfRgf(@Hi9^mP^FCK1eeu+d^9yc2KFd%RfSWJ0Gkv_>X2|Ukuw;a1 zylBRCxQ#W6k|22}$5YGK$1tJjWN~fvadm1g3zy*NyY6Y@P){~f&J$}=qXk=*L*~}^ zBHOtyBFq0OpoV=-aOz8$c#HKV6qiBZ0TW3=f227`X35{BCM>QUTOs-;90Q4@1Qfk~ zQ*LpO`f_ek%G-(sqqUu6LaqfS%W?c66m(s5-4x|=V&Q{By9otHy&&>LJ3G{*57ecv$T?A)}BO))F3Sv9K{%{+7bc~Za?*KIwF$BK3SSaFUOGwE1$Lq$D)T1!_Dz% zV_%)cFDpQP@)2U4!?v%4w}X9V^5C>UjS#FC`unCZOK@KqYa+_~70vzH*>8&(!N~St zE)b$Zn*eu$#!{mz;&D4yoq|d_K0ymOAO1_l7~S_rt~X)TtB@e>))e@c;{dqpP*o2mJW2-uU%|EY>}P}NsZ_;K~MOEU=;VnzeTTp z#Z{zPE$J7x9;!)J@yP7TxBMKN2GjagLWQ-1;HNh`&^A}q>@)D^%b#8pBH^< zY_EMMh!~h&QOm29(^2wGgUFfYY?Q651T-XPErLGiFu!ep8MIh+S@l+Ge))br7!T>f z;Y`9W%NXInmpyCTzx_*X7C3u^WGZ0eCI+f-<@h;ASjZVaLysLXe}%-;elxvGaY!*- z`f1t5JW5v!B4DvfXx1n)k}SBxF;~xvjZWY~@pD|jB7T9Ku3Nn@t#2^o+u$tODzw0v zC$w+Q_f0OB*<%fmdJnO<4D7Qd+3M7H2JG{%a|3l%8f0vYQ9N+$jOke7|Bn=G{eXr~ z8F;9#%&=YVsT`*a9LT016k2zC1su3V^-|R7R&@S&5!($WZ=@@tUHs^>?woZ=t41 zKcvT#mXI8sEP0UA5;F|O$AfHksM5U6IP?ixGhArWh0{*@E`WlWb*A5PKbF#31&NL} z1k#$j|8YT^dfzO181lTbQ7d!S^x5G?GjGprQ-8_$U}V?7v1iL{Qc9z(k`P*CNzA$A z-%~W6`5#53*595f$GMlw$}%l@pjD5j{$|Gj5Q~7@F@5#h@XYi>pA4K5=|(U`PJc|v z8tBsO|0a{2KQI^>_`h)h{tn6c%C>emLqWJ4=2Sg|n3-8igoAP#wdX|fY#ENYh94N) zDapN2wcT!(pu3J^8CQT@QwuccPw$ga?3Kz;I}7c=M>HLd)bdyQBhX9f0zES zaW^^M9C?j!__hh)Mho%edI#HOYMgVD`}B>jG1Z;r|E%Ht5*{;a8#?Jyxd4WLv@vTs zg7`K5EL+W#tZH9-EkghFm*eD&k{4ZaWmu7xlOM$#uLbCA)?zx9oXI?yrrw&eQRbvE z{sEy<;cYTrEty)&+`jA52cFj(%&pMcv&fhj|XOR zz+q(U-cvI<^j?PJsDaojFKAiffJLovjaZjbS%%%t4j%a^S82}xhoblnN93r$U5|#? zh&fWO$H8KVZi~fYuZhbI&Q{s@auE;K#gF68yRclje9x~3+^K2+X+NxqSaH;b>DaRP z44Oe-cQ&dx>3x>m_&oazkbMQ%z=kbln&YPpwGH#{lAo z%2(?~HgM*E@uqKO5$3Po*mgJx_}saUb{Wm##YVmX!7aIyP`@dAiw*B^edy&^cZ!X| zdaFZt!5~#WIbI^k{^j9Y?{xy(Hr_bBwDKZpast$dq5Vlb+J3w zDl8nU^b;12RR*Y@{<5}Fa#}C2HlGii;03cxvXRa<3NCvHMVii+7F~tmXor#xXlVNw z|0|O=N0lxDJiclb{g5Vh($A>ZN$Ax^ob>UWXmm$nMLpC5Y68A=o*SBTGkS!SvnAh3T{NA^K6CsC+SP@l4h%0mck)FlCFddIoB7+uOC`S1- zS1GrucNcV3etp_0JCU36CGx` zGT}F~p$E^5u7K!FLII-_J26P0HA!#5AscdjHyH&sM949)qZScNEB#>H7CHuf2RN)r zX;mg1;*(e+HR9Vk)Ek7xWHtxN-v$LRc|FKnU@@5sY}d=E$Mmr6XW2*#5bxxJ~QW%*ZjooADl7(^cmKX1eIf8=}OerLEh#{5n{s5 zGmO-W|6JNf0vdMLwM|r`a1&TuaCi4Fx;5ZAF?I!0r*R5>!awEmM^%2_?hKW$W74-F5sOG?@?(l;j^#gNisS!{Db{pJU{{{9>%KY`gG%X&-YcZ|C=F(&D^SH5 zjc@ZQ8p3rTWlhh68B8< zN<4g$oB&6Fn;DzxBDZSEJ9hR-sP2VCa)($>Q?pD3Ii_E3{oT0?cNKQWGt+f#_9gQ% z)$;K}e}4{X^PXKFL2jBgme9qVwW#BS{m?y;?q7Jp-+~8PYGM_`&(tK-5PG+f ztjlQ{6fU zJ*TV-tJ#{9u3^%2F4EaaAQ4>c8vkF{v8$Hs#A)kfdWByiQ-b0-oShOpy0n6XsqTt4 z*Uu@VmexDW?{uN!p_0`n?00euW#rHw`0OB^P!DBnq=-S+NR?qaf%mqROuBY&N6Z(F zzw7Voa~hQU ztfPzXM&h@2F*oX{#){j?B^|^ptv>GIX$_;TR)La0C8tXq3sP|}qp zj{VwlR`RN`)>HoIe-4Z=!Q_`ysp?9xYrMEXg+^A79OA=q-IgkN?N>VY7hsU0D$q_& z0SoXuf{g5We0WW?X=0deNs08;T?HHW-I5QCC3DzF5NB$fxhN#a4K@=B;S&}j)&_9K z!t@gE88PoBcgX=dE?oxCC=Xj!u{F)jtu<0|$0UNLuz<`my25JfjCanbdg|c(uGh== z>UnHeWhm6j8eb%`H> zCQ02&8qtKi*PFgLQ=SP}(=TW8takVGQs@&O8D~W{%lWeR8h)>C$I5n=xBDW*zWxCX zbhlOPYB)Y`PBCKu;Nv+!5$3^JYN6;Re>yxoQMtC=$HKWJSQmju&4f|ki z(0HQP+b$8T?c4p)<=^b3wlGu}5{)Nb9eM3*ZLE9>Pa(g(wt4YWdQ|27Cpwb|q^|IH z{`a19Yi1w#^_9(mrek3EL{&MOL~G5U7jd%GJuY;*n3giwo(A=w#*P?hph3&7MMJwr zKTR&OP>Il?YjH``D|48=daD-FgS?}=-j_KpU3B2B%3n6mY8E+9&9@~FJvNTn9QpHT z=II?rlO(qV!_L~KiRxGeuAIo30pwElGBLpgYw-!^uGR|JVr257k}r?#HvcsgrX2^2jq%-Y*EbSlZAuRDZP8 z-@BM8G`jTnziUDw*Jl1fea@IuUz4)l)Thx#IqZxB&oLt0+@P>z_5t@-M&Hl0;)pu` zn?~?-uTde}V~mtPW7}n==$OQVI}DU|>m$#{i-8{5>m!eYp1^X8;+4hWIG^HZmP8o) zZH6R`3qi|UKBJpzzPMBe7TkaS@ZgNLtPmG@>l1xmV8uX3h}>`Z)^KRVBNV%4?84*a z>ORHeH?+fd_%6fT3C*0X*}Sn#siJCtsRVz#@hrSo-Hgtw3muny6&KIwOqdivtoVA@ z8ce8f;)0(}@7Zfcj=;^MK0f|~8mF5LB=|cgtXDNQDwr8Oe*Z{<$9}*JShf%#B)K(D zmf%Dg9dw%@TXXeKeHCREcCfCiVZA{`+{=MWNYdgfKt%;*!=H+^D=SE+s$QDB=vs4v z+IbBE-f2iYl(8UhjBk4=K{s`Y?w!?GLK*GcACie-2ogV2KpPFEUsnBhx58Xf$qbY= z2AmN3EHv3kj*Cu-q)2wTpV{E#Dn9-;-Xhd0jXjj;TN7isEIOGlr8-T`IJ_G>Q7G-@ zXZz4v0oM6|UL7IiL3m42fuTdmKWCyraS)0cYK@2!YtW}l>7;`yuYOH4ss;4ZGCvmA z!h0KLSMrT-<@I%6B7st56;s67M+_;UnLFcxf4Z-`3vS}QI;n_yo+C#2Nbkp&AZ6y7 z4tjsW-=Gs1x9#~-7D)R4wlxGBx+OYt4yAwOQFk$zT3P)Keh1DI&iZ}EgBx=-MB_&u ze(i<4|4CC6W#LP(S_98l&xw$a$h{cxXRLY7(%GiEeKZ}pbB!BSC*84%dtMr>onY6m zBW-_B+(bSSZ*fIlOLV>SA#AOMU-93qb7lXkb=TOpH&60M9K62~JAosIlw|>@{ z5(B>GW{Y9uB0icd8C5M2sZH14qD4Pby^Qn92DBj&@GN8tA@3KL*UtPH^HTADMap#7 zN%!bsUn~%JfjNH9Ej`^Ox4o?HqH1g4MYK2Tr?AyDc{PhMNdOsvk`!}4!yuk_G zf>)m@>RMe)_p8voQ+Y{SF5Sy}>uT9%Rmdjq(o=x;Xgh}j-w^lMXAu-)aC>Xq9JcPz z@i;-*0WG^L*EWZ`O@I)uck`m~=?`H}$`7YcG$a zYs}>CiQ|G)x=V(l8+>yFja8E)Jo6)J9fEJ>Ob2+VPzCH5L|dG)UM~>*xOKh)Q`OkX>SyEq}@r0u_pm z?K@))?rjotZF7nQmPLls2wySOnt!)Z$>@#hvif|^2%!mMm7{r^j_kO*VhQ5$z20UY zE&^9B@Yt_RGBITh`I^D+Ip>;^Uw3DZn&Kq-)N-vIhe8$2v)OB12l7G7asn=)o>`~9 znF4kf?c%(&ipC*1pKWrAws`tWL_d&eaMh~4*=rne<)$7Y#tm`611z>a`BP6ks6O74 zD=r@Q68;phP7PB4yb9!+TYP=);K8r@dyD`UyS4Y>7?j%{0BQl$fa5=0#`WKdl3@DJ8W&wUjvQg+y$pC+sTgHzRW^i4aLIT(ke zF$FQdQh7IyISO!uPWTV-F;SMle6mO;w=;r&R{#HJmc=kY`U+9S%@)ZZ|A31<8IqUVsXY*Zrk(pLAv7^w!S;A-KLdGwRj z&_`^O{t+Pwvb7aXs;wU)RSNrMbRrkdU}z7uN_UXk=C6Y}Rre!gzw#}P5z*oHt&a|A zPStCKg$*ZTvebBjicS!BkUJA NIVokyYH^d0{|8puR@ndm literal 0 HcmV?d00001 diff --git a/oellm/main.py b/oellm/main.py index eea29583..f03c6935 100644 --- a/oellm/main.py +++ b/oellm/main.py @@ -5,6 +5,7 @@ from typer import rich_utils from oellm.config import EvalConfig +from oellm.push import push_results from oellm.results import collect_results from oellm.utils import _filter_warnings, _setup_logging @@ -460,6 +461,7 @@ def eval_command( app.command("schedule")(schedule_evals) app.command("eval")(eval_command) app.command("collect")(collect_results) +app.command("push")(push_results) app.command("list-tasks")(list_tasks) app.command("compare")(compare) app.command("doctor")(doctor) diff --git a/oellm/push.py b/oellm/push.py new file mode 100644 index 00000000..41d7ea77 --- /dev/null +++ b/oellm/push.py @@ -0,0 +1,318 @@ +"""Publish collected results to an ELLIOT dashboard (``oellm-eval push``). + +Runs on cluster login nodes, which have outbound HTTPS but accept no inbound +connections: results are pushed out, the dashboard never reaches in. Standard +library only, so it works in whatever environment already runs ``oellm-eval``. +""" + +from __future__ import annotations + +import http.client +import json +import logging +import os +import socket +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +ENVELOPE_NAME = "eval_results.json" +DEFAULT_TOKEN_FILE = "~/.config/oellm/dash_token" +TIMEOUT_S = 30 +BACKOFF_S = (1, 2, 4) # one entry per retry +_LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1"} + + +class PushError(Exception): + """A problem the user has to fix (configuration, not a transient failure).""" + + +@dataclass(frozen=True) +class PushOutcome: + path: Path + status: str # ingested | duplicate | dry-run | skipped | failed + rows: int = 0 + detail: str = "" + unreachable: bool = False + + @property + def ok(self) -> bool: + return self.status not in ("failed", "skipped") + + +def resolve_server(server: str | None) -> str: + url = (server or os.environ.get("OELLM_DASH_URL") or "").strip().rstrip("/") + if not url: + raise PushError("no dashboard address: pass --server or set OELLM_DASH_URL") + parts = urllib.parse.urlsplit(url) + try: + valid = parts.scheme in ("http", "https") and bool(parts.hostname) + valid = valid and parts.port != 0 + except ValueError: # malformed port + valid = False + if not valid: + raise PushError(f"not a valid dashboard address: {url!r}") + if parts.scheme == "http" and parts.hostname not in _LOCAL_HOSTS: + raise PushError( + f"refusing to send a token over plain http to {parts.hostname}; use https" + ) + return url + + +def resolve_token(token_file: str | None) -> str: + """--token-file, $OELLM_DASH_TOKEN_FILE, $OELLM_DASH_TOKEN, then the default + file. There is no option taking the token itself: command lines show up in + ``ps`` and shell history.""" + explicit = token_file or os.environ.get("OELLM_DASH_TOKEN_FILE") + if explicit: + return _read_token_file(Path(explicit).expanduser(), must_exist=True) + if os.environ.get("OELLM_DASH_TOKEN", "").strip(): + return os.environ["OELLM_DASH_TOKEN"].strip() + token = _read_token_file(Path(DEFAULT_TOKEN_FILE).expanduser(), must_exist=False) + if not token: + raise PushError( + "no dashboard token: put it in ~/.config/oellm/dash_token (chmod 600), " + "or set OELLM_DASH_TOKEN_FILE or OELLM_DASH_TOKEN" + ) + return token + + +def _read_token_file(path: Path, *, must_exist: bool) -> str: + if not path.is_file(): + if must_exist: + raise PushError(f"token file not found: {path}") + return "" + if path.stat().st_mode & 0o077: + logging.warning(f"{path} is readable by other users; run: chmod 600 {path}") + token = path.read_text().strip() + if not token and must_exist: + raise PushError(f"token file is empty: {path}") + return token + + +def find_envelopes(path: str | Path) -> list[Path]: + p = Path(path) + if p.is_dir(): + return sorted(p.rglob(ENVELOPE_NAME)) + if p.is_file(): + return [p] + raise PushError(f"no such file or directory: {p}") + + +def _load_envelope(path: Path) -> dict | None: + try: + data = json.loads(path.read_text()) + except (OSError, ValueError): + return None + if isinstance(data, dict) and isinstance(data.get("results"), list): + return data + return None + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """urllib would replay the Authorization header to wherever a redirect + points, and turn the POST into a GET. Neither is acceptable here.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.HTTPError( + req.full_url, code, f"redirected to {newurl}", headers, fp + ) + + +def _source_label(path: Path) -> str: + label = f"{socket.gethostname()}:{path.resolve()}" + return label.encode("ascii", "replace").decode() + + +def push_envelope( + path: Path, + server: str, + token: str, + *, + timeout: float = TIMEOUT_S, + sleep: Callable[[float], None] = time.sleep, +) -> PushOutcome: + """POST one envelope. Retries connection errors and 5xx; a 4xx is final.""" + data = _load_envelope(path) + if data is None: + return PushOutcome(path, "skipped", detail="not an eval_results envelope") + + from oellm import __version__ + + request = urllib.request.Request( + f"{server}/api/ingest", + data=json.dumps(data).encode(), + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "X-Elliot-Source": _source_label(path), + "User-Agent": f"oellm-eval/{__version__}", + }, + ) + opener = urllib.request.build_opener(_NoRedirect) + error, unreachable = "", False + for attempt in range(len(BACKOFF_S) + 1): + if attempt: + sleep(BACKOFF_S[attempt - 1]) + try: + with opener.open(request, timeout=timeout) as response: + raw = response.read() + except urllib.error.HTTPError as e: + error, unreachable = f"HTTP {e.code}: {_error_detail(e)}", False + if e.code < 500: + return PushOutcome(path, "failed", detail=error) + continue + except (urllib.error.URLError, http.client.HTTPException, OSError) as e: + reason = getattr(e, "reason", e) + if isinstance(reason, TimeoutError): + # Not retried: every retry would wait the full timeout again. + return PushOutcome( + path, + "failed", + detail=f"no answer from the dashboard within {timeout:g} s", + unreachable=True, + ) + error, unreachable = f"{type(e).__name__}: {reason}", True + continue + return _parse_reply(path, raw) + return PushOutcome( + path, + "failed", + detail=f"{error} (gave up after {len(BACKOFF_S)} retries)", + unreachable=unreachable, + ) + + +def _parse_reply(path: Path, raw: bytes) -> PushOutcome: + try: + body = json.loads(raw) + except ValueError: + body = None + if not isinstance(body, dict) or body.get("status") not in ("ingested", "duplicate"): + return PushOutcome( + path, + "failed", + detail="the reply did not come from an ELLIOT dashboard; check the address", + ) + rows = body.get("rows") + return PushOutcome(path, body["status"], rows if isinstance(rows, int) else 0) + + +def _error_detail(e: urllib.error.HTTPError) -> str: + if e.code == 401: + return "the dashboard rejected the token (wrong, or revoked)" + if 300 <= e.code < 400: + return f"{e.reason}; set the address to the final https location" + try: + return str(json.loads(e.read()).get("detail", e.reason)) + except (ValueError, AttributeError, OSError): + return str(e.reason) + + +def push_path( + path: str | Path, + *, + server: str | None = None, + token_file: str | None = None, + dry_run: bool = False, +) -> list[PushOutcome]: + url = resolve_server(server) + envelopes = find_envelopes(path) + if not envelopes: + raise PushError( + f"no {ENVELOPE_NAME} under {path}; collect writes it to the directory " + "it is run from, or next to --output-csv" + ) + if dry_run: + outcomes = [] + for f in envelopes: + data = _load_envelope(f) + outcomes.append( + PushOutcome(f, "dry-run", len(data["results"])) + if data + else PushOutcome(f, "skipped", detail="not an eval_results envelope") + ) + return outcomes + token = resolve_token(token_file) + outcomes = [] + for i, f in enumerate(envelopes): + outcome = push_envelope(f, url, token) + outcomes.append(outcome) + if outcome.unreachable: + outcomes += [ + PushOutcome(g, "failed", detail="not sent: the dashboard is unreachable") + for g in envelopes[i + 1 :] + ] + break + return outcomes + + +def _report(outcomes: list[PushOutcome], server: str) -> None: + for o in outcomes: + if o.status == "ingested": + logging.info(f"pushed {o.path}: {o.rows} rows") + elif o.status == "duplicate": + logging.info(f"already on the dashboard: {o.path}") + elif o.status == "dry-run": + logging.info(f"would push {o.path}: {o.rows} rows to {server}") + elif o.status == "skipped": + logging.error(f"skipped {o.path}: {o.detail}") + else: + logging.error(f"failed {o.path}: {o.detail}") + + +def push_results( + path: str, + *, + server: str | None = None, + token_file: str | None = None, + dry_run: bool = False, + verbose: bool = False, +) -> None: + """ + Push collected results to the ELLIOT dashboard. + + Run it on the login node after `collect`. Pushing the same results twice is + harmless: the dashboard recognises them and changes nothing. + + Args: + path: An eval_results.json file, or a directory searched recursively for them + server: Dashboard address, e.g. https://host/elliot-dashboard (default: $OELLM_DASH_URL) + token_file: File holding your upload token (default: $OELLM_DASH_TOKEN_FILE, + then $OELLM_DASH_TOKEN, then ~/.config/oellm/dash_token) + dry_run: Show what would be pushed; sends nothing and needs no token + verbose: Enable verbose logging + """ + from oellm.utils import _setup_logging + + _setup_logging(verbose) + try: + outcomes = push_path(path, server=server, token_file=token_file, dry_run=dry_run) + except PushError as e: + logging.error(str(e)) + raise SystemExit(2) from None + _report(outcomes, resolve_server(server)) + if not all(o.ok for o in outcomes): + raise SystemExit(1) + + +def push_after_collect(envelope: Path) -> bool: + """``collect --push``: publish the envelope just written. A push problem + must never cost the user their collected results, so nothing is raised.""" + try: + outcomes = push_path(envelope) + _report(outcomes, resolve_server(None)) + ok = all(o.ok for o in outcomes) + except Exception as e: + logging.warning(f"push failed: {e}") + ok = False + if not ok: + logging.warning( + f"results are saved locally; retry with: oellm-eval push {envelope}" + ) + return ok diff --git a/oellm/results.py b/oellm/results.py index 715d2e78..3a4ff706 100644 --- a/oellm/results.py +++ b/oellm/results.py @@ -401,6 +401,7 @@ def collect_results( *, check: bool = False, fetch_all_metrics: bool = False, + push: bool = False, verbose: bool = False, ) -> None: """ @@ -412,6 +413,7 @@ def collect_results( check: Check for missing evaluations and create a missing jobs CSV fetch_all_metrics: Emit one row per numeric metric the engine reported instead of only the task's primary metric + push: Also push the collected results to the dashboard (see `oellm-eval push`) verbose: Enable verbose logging """ _setup_logging(verbose) @@ -847,6 +849,11 @@ def _emit(model: str, task: str, n_shot, pairs: list[tuple[str, float]]) -> None logging.info(f"Extracted {len(df)} evaluation results") + if push: + from oellm.push import push_after_collect + + push_after_collect(json_path) + if verbose: logging.info("Summary:") logging.info(f"Unique models: {df['model_name'].nunique()}") diff --git a/tests/test_push.py b/tests/test_push.py new file mode 100644 index 00000000..30c64471 --- /dev/null +++ b/tests/test_push.py @@ -0,0 +1,367 @@ +"""`oellm-eval push` and `collect --push`, against a fake dashboard on localhost.""" + +import inspect +import json +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +from oellm import push +from oellm.push import PushError, push_envelope, push_path, push_results + +ENVELOPE = { + "version": "1.2", + "generated_at": "2026-09-21T10:00:00+00:00", + "runs": [{"submitted_by": "ivan", "limit": None}], + "results": [ + { + "model": "m", + "task": "copa", + "n_shot": 0, + "metric": "acc,none", + "performance": 0.5, + } + ], +} + + +class FakeDashboard: + """Answers POSTs from a queue of (status, body); records what it received.""" + + def __init__(self): + self.requests: list[dict] = [] + self.responses: list[tuple[int, dict, dict]] = [] + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + outer.requests.append( + {"path": self.path, "headers": dict(self.headers), "body": body} + ) + status, payload, headers = ( + outer.responses.pop(0) + if outer.responses + else (200, {"status": "ingested", "rows": 1}, {}) + ) + self.send_response(status) + for k, v in headers.items(): + self.send_header(k, v) + self.end_headers() + self.wfile.write( + payload.encode() + if isinstance(payload, str) + else json.dumps(payload).encode() + ) + + do_GET = do_POST + + def log_message(self, *args): + pass + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}" + + def close(self): + self.httpd.shutdown() + self.httpd.server_close() + + +@pytest.fixture +def dashboard(): + server = FakeDashboard() + yield server + server.close() + + +@pytest.fixture +def envelope(tmp_path): + path = tmp_path / "run" / "eval_results.json" + path.parent.mkdir() + path.write_text(json.dumps(ENVELOPE, indent=2)) + return path + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch, tmp_path): + for var in ("OELLM_DASH_URL", "OELLM_DASH_TOKEN", "OELLM_DASH_TOKEN_FILE"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + + +class TestRequest: + def test_sends_the_envelope_with_token_and_origin(self, dashboard, envelope): + outcome = push_envelope(envelope, dashboard.url, "edt_secret") + assert (outcome.status, outcome.rows) == ("ingested", 1) + (req,) = dashboard.requests + assert req["path"] == "/api/ingest" + assert json.loads(req["body"]) == ENVELOPE + assert req["headers"]["Authorization"] == "Bearer edt_secret" + assert req["headers"]["X-Elliot-Source"].endswith(str(envelope.resolve())) + assert req["headers"]["User-Agent"].startswith("oellm-eval/") + + def test_prefix_is_kept_in_the_request_path(self, dashboard, envelope, monkeypatch): + """The hosted dashboard lives under /elliot-dashboard behind a proxy.""" + monkeypatch.setenv("OELLM_DASH_TOKEN", "edt_x") + push_path(envelope, server=dashboard.url + "/elliot-dashboard/") + assert dashboard.requests[0]["path"] == "/elliot-dashboard/api/ingest" + + def test_duplicate_counts_as_success(self, dashboard, envelope): + dashboard.responses.append((200, {"status": "duplicate", "rows": 0}, {})) + outcome = push_envelope(envelope, dashboard.url, "t") + assert outcome.status == "duplicate" and outcome.ok + + +class TestFailures: + def test_server_errors_are_retried_with_backoff(self, dashboard, envelope): + dashboard.responses += [(500, {}, {}), (503, {}, {})] + waits: list[float] = [] + outcome = push_envelope(envelope, dashboard.url, "t", sleep=waits.append) + assert outcome.status == "ingested" + assert len(dashboard.requests) == 3 and waits == [1, 2] + + def test_gives_up_after_the_last_retry(self, envelope): + waits: list[float] = [] + outcome = push_envelope( + envelope, "http://127.0.0.1:9", "t", timeout=2, sleep=waits.append + ) + assert outcome.status == "failed" and not outcome.ok and outcome.unreachable + assert waits == [1, 2, 4] and "gave up" in outcome.detail + + @pytest.mark.parametrize( + "reply", + [[1, 2], {"ok": True}, "sign in"], + ids=["json-array", "json-object-without-status", "html-page"], + ) + def test_a_reply_not_from_the_dashboard_fails(self, dashboard, envelope, reply): + dashboard.responses.append((200, reply, {})) + waits: list[float] = [] + outcome = push_envelope(envelope, dashboard.url, "t", sleep=waits.append) + assert outcome.status == "failed" and "dashboard" in outcome.detail + assert len(dashboard.requests) == 1 and waits == [] + + def test_a_timeout_is_not_retried(self, envelope): + srv = socket.socket() + srv.bind(("127.0.0.1", 0)) + srv.listen(8) + accepted: list[socket.socket] = [] + + def accept_forever(): + try: + while True: + accepted.append(srv.accept()[0]) + except OSError: + pass + + threading.Thread(target=accept_forever, daemon=True).start() + waits: list[float] = [] + outcome = push_envelope( + envelope, + f"http://127.0.0.1:{srv.getsockname()[1]}", + "t", + timeout=1, + sleep=waits.append, + ) + srv.close() + for conn in accepted: + conn.close() + assert outcome.status == "failed" and outcome.unreachable + assert len(accepted) == 1 and waits == [] + + def test_an_unreachable_dashboard_stops_the_remaining_pushes( + self, tmp_path, monkeypatch + ): + monkeypatch.setenv("OELLM_DASH_TOKEN", "t") + monkeypatch.setattr(push, "BACKOFF_S", ()) + for name in ("a", "b", "c"): + d = tmp_path / "out" / name + d.mkdir(parents=True) + (d / "eval_results.json").write_text(json.dumps(ENVELOPE)) + outcomes = push_path(tmp_path / "out", server="http://127.0.0.1:9") + assert [o.status for o in outcomes] == ["failed"] * 3 + assert outcomes[0].unreachable + assert all("not sent" in o.detail for o in outcomes[1:]) + + def test_client_errors_are_final(self, dashboard, envelope): + dashboard.responses.append((401, {"detail": "missing or invalid token"}, {})) + waits: list[float] = [] + outcome = push_envelope(envelope, dashboard.url, "bad", sleep=waits.append) + assert outcome.status == "failed" and "token" in outcome.detail + assert len(dashboard.requests) == 1 and waits == [] + + def test_validation_error_from_the_dashboard_is_shown(self, dashboard, envelope): + dashboard.responses.append((422, {"detail": "runs must be a list"}, {})) + assert "runs must be a list" in push_envelope(envelope, dashboard.url, "t").detail + + def test_redirects_are_not_followed(self, dashboard, envelope): + """urllib would resend the token to the redirect target.""" + dashboard.responses.append((302, {}, {"Location": dashboard.url + "/elsewhere"})) + outcome = push_envelope( + envelope, dashboard.url, "edt_secret", sleep=lambda s: None + ) + assert outcome.status == "failed" and "redirected" in outcome.detail + assert [r["path"] for r in dashboard.requests] == ["/api/ingest"] + + def test_non_envelope_file_is_skipped_not_sent(self, dashboard, tmp_path): + other = tmp_path / "results.json" + other.write_text(json.dumps({"results": {"copa": {"acc": 0.5}}})) + assert push_envelope(other, dashboard.url, "t").status == "skipped" + assert dashboard.requests == [] + + +class TestConfiguration: + def test_plain_http_to_a_remote_host_is_refused(self): + with pytest.raises(PushError, match="plain http"): + push.resolve_server("http://dashboard.example.org/elliot") + assert push.resolve_server("https://dashboard.example.org/elliot/") == ( + "https://dashboard.example.org/elliot" + ) + + def test_a_malformed_port_is_a_configuration_error(self): + with pytest.raises(PushError, match="not a valid dashboard address"): + push.resolve_server("https://dashboard.example.org:abc/elliot") + + def test_missing_address_is_a_clear_error(self): + with pytest.raises(PushError, match="OELLM_DASH_URL"): + push.resolve_server(None) + + def test_token_sources_in_order(self, tmp_path, monkeypatch): + default = Path(tmp_path / "home/.config/oellm/dash_token") + default.parent.mkdir(parents=True) + default.write_text("from-default\n") + default.chmod(0o600) + assert push.resolve_token(None) == "from-default" + + monkeypatch.setenv("OELLM_DASH_TOKEN", "from-env") + assert push.resolve_token(None) == "from-env" + + env_file = tmp_path / "env_token" + env_file.write_text("from-env-file") + env_file.chmod(0o600) + monkeypatch.setenv("OELLM_DASH_TOKEN_FILE", str(env_file)) + assert push.resolve_token(None) == "from-env-file" + + flag_file = tmp_path / "flag_token" + flag_file.write_text(" from-flag \n") + flag_file.chmod(0o600) + assert push.resolve_token(str(flag_file)) == "from-flag" + + def test_missing_token_is_a_clear_error(self, tmp_path): + with pytest.raises(PushError, match="dash_token"): + push.resolve_token(None) + with pytest.raises(PushError, match="not found"): + push.resolve_token(str(tmp_path / "nope")) + + def test_the_token_can_never_be_passed_on_the_command_line(self): + """argv is visible in `ps` and shell history.""" + assert "token" not in inspect.signature(push_results).parameters + assert set(inspect.signature(push_results).parameters) == { + "path", + "server", + "token_file", + "dry_run", + "verbose", + } + + +class TestCommand: + def test_directory_is_searched_recursively(self, dashboard, tmp_path, monkeypatch): + monkeypatch.setenv("OELLM_DASH_TOKEN", "t") + for name in ("a", "b/nested"): + d = tmp_path / "out" / name + d.mkdir(parents=True) + (d / "eval_results.json").write_text(json.dumps(ENVELOPE)) + (tmp_path / "out" / "a" / "provenance.json").write_text("{}") + outcomes = push_path(tmp_path / "out", server=dashboard.url) + assert [o.status for o in outcomes] == ["ingested", "ingested"] + assert len(dashboard.requests) == 2 + + def test_dry_run_sends_nothing_and_needs_no_token(self, dashboard, envelope): + (outcome,) = push_path(envelope, server=dashboard.url, dry_run=True) + assert (outcome.status, outcome.rows) == ("dry-run", 1) + assert dashboard.requests == [] + + def test_a_file_that_is_not_an_envelope_fails_the_command( + self, dashboard, tmp_path, monkeypatch + ): + monkeypatch.setenv("OELLM_DASH_TOKEN", "t") + csv = tmp_path / "eval_results.csv" + csv.write_text("model_name,task\nm,copa\n") + with pytest.raises(SystemExit) as failed: + push_results(str(csv), server=dashboard.url) + assert failed.value.code == 1 and dashboard.requests == [] + + def test_nothing_to_push_is_an_error(self, dashboard, tmp_path): + with pytest.raises(PushError, match="collect"): + push_path(tmp_path, server=dashboard.url) + + def test_exit_codes(self, dashboard, envelope, monkeypatch): + monkeypatch.setenv("OELLM_DASH_TOKEN", "t") + push_results(str(envelope), server=dashboard.url) # success: returns + + dashboard.responses.append((401, {}, {})) + with pytest.raises(SystemExit) as failed: + push_results(str(envelope), server=dashboard.url) + assert failed.value.code == 1 + + with pytest.raises(SystemExit) as misconfigured: + push_results(str(envelope), server=None) + assert misconfigured.value.code == 2 + + def test_registered_as_a_cli_command(self): + from oellm.main import app + + names = {c.name or c.callback.__name__ for c in app.registered_commands} + assert "push" in names + + +LM_EVAL_RESULT = { + "model_name": "EleutherAI/pythia-70m", + "results": {"copa": {"acc,none": 0.5, "alias": "copa"}}, + "configs": {"copa": {"num_fewshot": 0}}, +} + + +class TestCollectPush: + def _run_dir(self, tmp_path: Path) -> Path: + results = tmp_path / "run" / "results" + results.mkdir(parents=True) + (results / "abc.json").write_text(json.dumps(LM_EVAL_RESULT)) + return tmp_path / "run" + + def test_collect_push_publishes_what_it_wrote(self, dashboard, tmp_path, monkeypatch): + from oellm.results import collect_results + + monkeypatch.setenv("OELLM_DASH_URL", dashboard.url) + monkeypatch.setenv("OELLM_DASH_TOKEN", "t") + run = self._run_dir(tmp_path) + collect_results(str(run), str(run / "eval_results.csv"), push=True) + + (req,) = dashboard.requests + sent = json.loads(req["body"]) + assert sent == json.loads((run / "eval_results.json").read_text()) + assert sent["results"][0]["task"] == "copa" + + def test_a_failed_push_never_fails_collect(self, tmp_path, monkeypatch): + from oellm.results import collect_results + + monkeypatch.setenv("OELLM_DASH_URL", "https://dashboard.invalid") + monkeypatch.setattr(push, "BACKOFF_S", ()) + run = self._run_dir(tmp_path) + collect_results(str(run), str(run / "eval_results.csv"), push=True) # no token + assert (run / "eval_results.csv").exists() and ( + run / "eval_results.json" + ).exists() + + def test_without_the_flag_nothing_is_sent(self, dashboard, tmp_path, monkeypatch): + from oellm.results import collect_results + + monkeypatch.setenv("OELLM_DASH_URL", dashboard.url) + monkeypatch.setenv("OELLM_DASH_TOKEN", "t") + run = self._run_dir(tmp_path) + collect_results(str(run), str(run / "eval_results.csv")) + assert dashboard.requests == [] From 64a33c015ec34577253140184279fb829933fbd3 Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Sun, 27 Sep 2026 16:34:16 +0200 Subject: [PATCH 41/44] [Base] Keep lm-eval 0.4.12 name for INCLUDE North Macedonian --- oellm/resources/task-groups.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oellm/resources/task-groups.yaml b/oellm/resources/task-groups.yaml index 42ef011e..59da246d 100644 --- a/oellm/resources/task-groups.yaml +++ b/oellm/resources/task-groups.yaml @@ -291,7 +291,9 @@ task_groups: subset: Italian - task: include_base_44_lithuanian subset: Lithuanian - - task: include_base_44_north_macedonian + # lm-eval 0.4.12, which we pin, names this task with a space; 0.4.13 + # renamed it to include_base_44_north_macedonian. Flip it with wsc273. + - task: include_base_44_north macedonian subset: North Macedonian - task: include_base_44_polish subset: Polish From da04b0cc1ebc5f9389fbcaab29988b46775ea5ec Mon Sep 17 00:00:00 2001 From: Ivan Slobozhan Date: Sun, 27 Sep 2026 16:34:21 +0200 Subject: [PATCH 42/44] [Base] README: link to the dashboard site --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 29f73ef0..12e1caa3 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ A multimodal evaluation framework for scheduling LLM and VLM evaluations across ## Results Dashboard -`oellm-eval collect --push` sends results from the login node to the [ELLIOT dashboard](https://github.com/elliot-project/elliot-eval-dashboard), so results from all clusters end up in one place. +`oellm-eval collect --push` sends results from the login node to the [ELLIOT dashboard](http://test.openml.org/elliot-dashboard), so results from all clusters end up in one place.