Fmo/laiser local

backend/Makefile

@@ -70,6 +70,9 @@ requirements: clean_tox piptools ## install development environment requirements
 	# So that the plugin entrypoints are installed and loaded correctly.
 	pip install -e .
 
+requirements-laiser: ## install optional LAiSER local extractor dependencies (CPU mode)
+	pip install -r requirements/laiser.in
+
 test: clean ## run tests in the current virtualenv
 	pytest
 

backend/openedx_ai_badges/processors/badge_processor.py

@@ -3,13 +3,24 @@
 """
 import json
 import logging
+import time
+from functools import lru_cache
 from pathlib import Path
 
+import requests
+from django.conf import settings
 from openedx_ai_extensions.processors import LLMProcessor
 
 logger = logging.getLogger(__name__)
 
 
+@lru_cache(maxsize=4)
+def _get_laiser_extractor(model_id, hf_token, use_gpu):
+    """Return a cached Skill_Extractor instance keyed by (model_id, hf_token, use_gpu)."""
+    from laiser.skill_extractor import Skill_Extractor  # pylint: disable=import-error,import-outside-toplevel
+    return Skill_Extractor(AI_MODEL_ID=model_id, HF_TOKEN=hf_token, use_gpu=use_gpu)
+
+
 class BaseBadgeLLMProcessor(LLMProcessor):
     """Base processor for badge-related LLM tasks."""
     schema_filename = None
@@ -97,3 +108,179 @@ def generate_skills(self):
         prompt = self.fill_prompt(prompt)
         result = self._call_completion_wrapper(system_role=prompt)
         return result
+
+    def generate_skills_laiser_local(self):
+        """
+        Extract skills from course context using the LAiSER library running in-process.
+
+        Initializes Skill_Extractor once per (model_id, hf_token, use_gpu) combination
+        and caches it for subsequent calls. Wraps the context in a DataFrame, runs
+        extraction, enriches results with ESCO metadata, and normalizes to the internal
+        skills alignment format.
+        """
+        try:
+            import pandas as pd  # pylint: disable=import-outside-toplevel
+        except ImportError as exc:
+            logger.error("LAiSER local: pandas not installed: %s", exc)
+            return {"error": f"pandas not available: {exc}"}
+
+        model_id = self.config.get("model_id") or getattr(settings, "LAISER_MODEL_ID", "")
+        hf_token = self.config.get("hf_token") or getattr(settings, "LAISER_HF_TOKEN", "")
+        use_gpu = self.config.get("use_gpu", False)
+        top_k = self.config.get("top_k", 10)
+
+        if not model_id:
+            logger.error("LAiSER local: LAISER_MODEL_ID is not configured")
+            return {"error": "LAISER_MODEL_ID is not configured"}
+
+        try:
+            extractor = _get_laiser_extractor(model_id, hf_token, use_gpu)
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            logger.error("LAiSER local: failed to initialize extractor: %s", exc)
+            return {"error": f"LAiSER extractor initialization failed: {exc}"}
+
+        data = pd.DataFrame({"id": ["badge_1"], "description": [self.context or ""]})
+
+        try:
+            result_df = extractor.extractor(
+                data=data,
+                id_column="id",
+                text_columns=["description"],
+                input_type="job_desc",
+                top_k=top_k,
+                levels=False,
+                warnings=False,
+            )
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            logger.error("LAiSER local: extraction failed: %s", exc)
+            return {"error": f"LAiSER extraction failed: {exc}"}
+
+        raw_skills = result_df.to_dict("records") if isinstance(result_df, pd.DataFrame) else []
+
+        skills = [self._normalize_laiser_local_skill(s, extractor) for s in raw_skills]
+        return {"response": json.dumps({"skills": skills}), "status": "success"}
+
+    def generate_skills_laiser_api(self):
+        """
+        Submit course context to the LAiSER API and poll for extracted skills.
+
+        Resolves base_url and api_key from processor config or Django settings.
+        POSTs context to /laiser, polls /result until a terminal state, then
+        normalizes the result array into the internal skills alignment format.
+        """
+        base_url = self.config.get("base_url") or getattr(settings, "LAISER_API_BASE_URL", "")
+        api_key = self.config.get("api_key") or getattr(settings, "LAISER_API_KEY", "")
+        timeout = self.config.get("timeout_seconds", getattr(settings, "LAISER_API_TIMEOUT_SECONDS", 90))
+        poll_interval = self.config.get(
+            "poll_interval_seconds", getattr(settings, "LAISER_API_POLL_INTERVAL_SECONDS", 2)
+        )
+
+        if not base_url or not api_key:
+            logger.error("LAiSER API is not configured. Check LAISER_API_BASE_URL or LAISER_API_KEY")
+            return {"error": f"LAiSER API incorrectly configured"}
+
+        try:
+            submit_response = requests.post(
+                f"{base_url}/laiser",
+                json={"inputText": self.context},
+                headers={"x-api-key": api_key, "Content-Type": "application/json"},
+                timeout=30,
+            )
+            submit_response.raise_for_status()
+            submit_data = submit_response.json()
+        except requests.exceptions.RequestException as exc:
+            logger.error("LAiSER API submit failed: %s", exc)
+            return {"error": str(exc)}
+        except ValueError as exc:
+            logger.error("LAiSER API submit returned non-JSON: %s", exc)
+            return {"error": f"Invalid JSON from LAiSER submit: {exc}"}
+
+        job_id = submit_data.get("jobId")
+        if not job_id:
+            logger.error("LAiSER API submit response missing jobId: %s", submit_data)
+            return {"error": "No jobId in LAiSER submit response"}
+
+        result = self._poll_laiser_job(base_url, api_key, job_id, timeout, poll_interval)
+        if "error" in result:
+            return result
+
+        skills = [self._normalize_laiser_skill(s) for s in result.get("result", [])]
+        return {"response": json.dumps({"skills": skills}), "status": "success"}
+
+    def _poll_laiser_job(self, base_url, api_key, job_id, timeout, poll_interval):
+        """Poll GET /result until the job reaches a terminal state or timeout."""
+        elapsed = 0
+        while elapsed < timeout:
+            time.sleep(poll_interval)
+            elapsed += poll_interval
+
+            try:
+                response = requests.get(
+                    f"{base_url}/result",
+                    params={"jobId": job_id},
+                    headers={"x-api-key": api_key},
+                    timeout=30,
+                )
+                response.raise_for_status()
+                data = response.json()
+            except requests.exceptions.RequestException as exc:
+                logger.error("LAiSER API poll failed (jobId=%s): %s", job_id, exc)
+                return {"error": str(exc)}
+            except ValueError as exc:
+                logger.error("LAiSER API poll returned non-JSON (jobId=%s): %s", job_id, exc)
+                return {"error": f"Invalid JSON from LAiSER poll: {exc}"}
+
+            if data.get("status") in ("QUEUED", "RUNNING"):
+                continue
+
+            return data
+
+        logger.error("LAiSER API timed out after %ds, jobId=%s", timeout, job_id)
+        return {"error": f"LAiSER API timed out after {timeout} seconds"}
+
+    @staticmethod
+    def _normalize_laiser_skill(skill: dict) -> dict:
+        """Map a LAiSER API result entry to the internal skills alignment format."""
+        source = (skill.get("Taxonomy Source") or "").lower()
+        target_type_map = {
+            "esco": "ESCO:Skill",
+            "ukos": "UKOS:Skill",
+            "onet_tech": "ONET:Skill",
+        }
+        return {
+            "type": "Alignment",
+            "skill_tag": skill.get("Raw Skill", ""),
+            "target_name": skill.get("Taxonomy Skill", ""),
+            "target_description": skill.get("Taxonomy Description", ""),
+            "target_url": skill.get("Source URL", ""),
+            "target_type": target_type_map.get(source, source),
+            "correlation_coefficient": skill.get("Correlation Coefficient", 0),
+            "task_abilities": [],
+            "knowledge_required": [],
+        }
+
+    @staticmethod
+    def _normalize_laiser_local_skill(skill: dict, extractor) -> dict:
+        """Map a LAiSER local extractor result row to the internal skills alignment format."""
+        raw_skill = skill.get("Raw Skill", "")
+
+        target_description = ""
+        target_url = ""
+        if raw_skill and getattr(extractor, "esco_df", None) is not None:
+            match = extractor.esco_df[extractor.esco_df["preferredLabel"] == raw_skill]
+            if not match.empty:
+                row = match.iloc[0]
+                target_description = row.get("description", "")
+                target_url = row.get("conceptUri", "")
+
+        return {
+            "type": "Alignment",
+            "skill_tag": skill.get("Skill Tag", ""),
+            "target_name": raw_skill,
+            "target_description": target_description,
+            "target_url": target_url,
+            "target_type": "ESCO:Skill",
+            "correlation_coefficient": skill.get("Correlation Coefficient", 0),
+            "task_abilities": skill.get("Task Abilities", []),
+            "knowledge_required": skill.get("Knowledge Required", []),
+        }

backend/openedx_ai_badges/settings/common.py

@@ -37,3 +37,17 @@ def plugin_settings(settings):
     settings.MIT_SLM_OLLAMA_URL = ""
     settings.MIT_SLM_OLLAMA_TOKEN = ""
     settings.MIT_DCC_BADGE_API_HEALTH_URL = "http://mit-slm:8000/health"
+
+    # -------------------------
+    # LAiSER API
+    # -------------------------
+    settings.LAISER_API_BASE_URL = ""
+    settings.LAISER_API_KEY = ""
+    settings.LAISER_API_TIMEOUT_SECONDS = 90
+    settings.LAISER_API_POLL_INTERVAL_SECONDS = 2
+
+    # -------------------------
+    # LAiSER local
+    # -------------------------
+    settings.LAISER_MODEL_ID = ""
+    settings.LAISER_HF_TOKEN = ""

backend/openedx_ai_badges/settings/production.py

@@ -31,3 +31,31 @@ def plugin_settings(settings):
         settings.MIT_DCC_BADGE_API_HEALTH_URL = settings.ENV_TOKENS.get(
             "MIT_DCC_BADGE_API_HEALTH_URL", settings.MIT_DCC_BADGE_API_HEALTH_URL
         )
+
+    # -------------------------
+    # LAiSER API
+    # -------------------------
+    if hasattr(settings, "ENV_TOKENS"):
+        settings.LAISER_API_BASE_URL = settings.ENV_TOKENS.get(
+            "LAISER_API_BASE_URL", settings.LAISER_API_BASE_URL
+        )
+        settings.LAISER_API_KEY = settings.ENV_TOKENS.get(
+            "LAISER_API_KEY", settings.LAISER_API_KEY
+        )
+        settings.LAISER_API_TIMEOUT_SECONDS = settings.ENV_TOKENS.get(
+            "LAISER_API_TIMEOUT_SECONDS", settings.LAISER_API_TIMEOUT_SECONDS
+        )
+        settings.LAISER_API_POLL_INTERVAL_SECONDS = settings.ENV_TOKENS.get(
+            "LAISER_API_POLL_INTERVAL_SECONDS", settings.LAISER_API_POLL_INTERVAL_SECONDS
+        )
+
+    # -------------------------
+    # LAiSER local
+    # -------------------------
+    if hasattr(settings, "ENV_TOKENS"):
+        settings.LAISER_MODEL_ID = settings.ENV_TOKENS.get(
+            "LAISER_MODEL_ID", settings.LAISER_MODEL_ID
+        )
+        settings.LAISER_HF_TOKEN = settings.ENV_TOKENS.get(
+            "LAISER_HF_TOKEN", settings.LAISER_HF_TOKEN
+        )

backend/openedx_ai_badges/workflows/orchestrators.py

@@ -234,7 +234,8 @@ def regenerate(self, input_data):
         }
 
         if skills_requested:
-            self._set_status_message("Generating skills alignment...")
+            skills_fn = self.profile.processor_config.get("SkillsProcessor", {}).get("function", "generate_skills")
+            self._set_status_message(f'Generating skills alignment using "{skills_fn}"...')
             skills = self._get_skills(course_context, input_data, regenerate=True)
             if isinstance(skills, dict) and 'error' in skills:
                 return skills
@@ -442,7 +443,8 @@ def run(self, input_data):
         }
 
         if skills_enabled:
-            self._set_status_message("Generating skills alignment...")
+            skills_fn = self.profile.processor_config.get("SkillsProcessor", {}).get("function", "generate_skills")
+            self._set_status_message(f'Generating skills alignment using "{skills_fn}"...')
             skills = self._get_skills(course_context, input_data)
             if isinstance(skills, dict) and 'error' in skills:
                 return skills

backend/openedx_ai_badges/workflows/profiles/badges_base.json

@@ -10,8 +10,9 @@
           "provider": "openai",
       },
       "SkillsProcessor": {
-          "function": "generate_skills",
-          "provider": "openai",
+          "function": "generate_skills_laiser_api",
+          // "function": "generate_skills",
+          // "provider": "openai",
       }
   },
   "actuator_config": {

backend/requirements/laiser.in

@@ -0,0 +1,6 @@
+# Optional requirements for running the LAiSER local skill extractor.
+# Install with: pip install -r requirements/laiser.txt
+# or: make requirements-laiser
+
+laiser[cpu]  # LAiSER local skill extraction (CPU mode)
+pandas       # Required by the LAiSER extractor

tutor/openedx_ai_badges/patches/openedx-common-settings

@@ -1,3 +1,5 @@
+LAISER_API_BASE_URL = "{{ LAISER_API_BASE_URL }}"
+LAISER_API_KEY = "{{ LAISER_API_KEY }}"
 MIT_SLM_OLLAMA_URL = "{{ MIT_SLM_OLLAMA_URL }}"
 MIT_SLM_OLLAMA_TOKEN = "{{ MIT_SLM_OLLAMA_TOKEN }}"
 MIT_DCC_BADGE_API_HEALTH_URL = "{{ MIT_DCC_BADGE_API_HEALTH_URL }}"

tutor/openedx_ai_badges/plugin.py

@@ -53,6 +53,8 @@ def _mount_plugin(mounts, path):
 
 hooks.Filters.CONFIG_DEFAULTS.add_items(
     [
+        ("LAISER_API_BASE_URL", ""),
+        ("LAISER_API_KEY", ""),
         ("RUN_MIT_SLM", False),
         ("MIT_SLM_DOCKER_IMAGE", "felipemontoya/dcc-mit-badge-api:latest"),
         ("MIT_SLM_OLLAMA_URL", "https://felipemontoya-mit-dcc-ollama.hf.space/api/generate"),