From c879ce8dc879a02d72dbcacd0f65b5be783ab9cd Mon Sep 17 00:00:00 2001 From: Felipe Montoya Date: Tue, 7 Apr 2026 22:45:58 -0500 Subject: [PATCH 1/2] feat: using the laiser API to create skills --- .../processors/badge_processor.py | 101 ++++++++++++++++++ backend/openedx_ai_badges/settings/common.py | 8 ++ .../openedx_ai_badges/settings/production.py | 17 +++ .../workflows/orchestrators.py | 6 +- .../workflows/profiles/badges_base.json | 5 +- .../patches/openedx-common-settings | 2 + tutor/openedx_ai_badges/plugin.py | 2 + 7 files changed, 137 insertions(+), 4 deletions(-) diff --git a/backend/openedx_ai_badges/processors/badge_processor.py b/backend/openedx_ai_badges/processors/badge_processor.py index c220cc8..29c0fdb 100644 --- a/backend/openedx_ai_badges/processors/badge_processor.py +++ b/backend/openedx_ai_badges/processors/badge_processor.py @@ -3,8 +3,11 @@ """ import json import logging +import time from pathlib import Path +import requests +from django.conf import settings from openedx_ai_extensions.processors import LLMProcessor logger = logging.getLogger(__name__) @@ -97,3 +100,101 @@ def generate_skills(self): prompt = self.fill_prompt(prompt) result = self._call_completion_wrapper(system_role=prompt) return result + def generate_skills_laiser_api(self): + """ + Submit course context to the LAiSER API and poll for extracted skills. + + Resolves base_url and api_key from processor config or Django settings. + POSTs context to /laiser, polls /result until a terminal state, then + normalizes the result array into the internal skills alignment format. + """ + base_url = self.config.get("base_url") or getattr(settings, "LAISER_API_BASE_URL", "") + api_key = self.config.get("api_key") or getattr(settings, "LAISER_API_KEY", "") + timeout = self.config.get("timeout_seconds", getattr(settings, "LAISER_API_TIMEOUT_SECONDS", 90)) + poll_interval = self.config.get( + "poll_interval_seconds", getattr(settings, "LAISER_API_POLL_INTERVAL_SECONDS", 2) + ) + + if not base_url or not api_key: + logger.error("LAiSER API is not configured. Check LAISER_API_BASE_URL or LAISER_API_KEY") + return {"error": f"LAiSER API incorrectly configured"} + + try: + submit_response = requests.post( + f"{base_url}/laiser", + json={"inputText": self.context}, + headers={"x-api-key": api_key, "Content-Type": "application/json"}, + timeout=30, + ) + submit_response.raise_for_status() + submit_data = submit_response.json() + except requests.exceptions.RequestException as exc: + logger.error("LAiSER API submit failed: %s", exc) + return {"error": str(exc)} + except ValueError as exc: + logger.error("LAiSER API submit returned non-JSON: %s", exc) + return {"error": f"Invalid JSON from LAiSER submit: {exc}"} + + job_id = submit_data.get("jobId") + if not job_id: + logger.error("LAiSER API submit response missing jobId: %s", submit_data) + return {"error": "No jobId in LAiSER submit response"} + + result = self._poll_laiser_job(base_url, api_key, job_id, timeout, poll_interval) + if "error" in result: + return result + + skills = [self._normalize_laiser_skill(s) for s in result.get("result", [])] + return {"response": json.dumps({"skills": skills}), "status": "success"} + + def _poll_laiser_job(self, base_url, api_key, job_id, timeout, poll_interval): + """Poll GET /result until the job reaches a terminal state or timeout.""" + elapsed = 0 + while elapsed < timeout: + time.sleep(poll_interval) + elapsed += poll_interval + + try: + response = requests.get( + f"{base_url}/result", + params={"jobId": job_id}, + headers={"x-api-key": api_key}, + timeout=30, + ) + response.raise_for_status() + data = response.json() + except requests.exceptions.RequestException as exc: + logger.error("LAiSER API poll failed (jobId=%s): %s", job_id, exc) + return {"error": str(exc)} + except ValueError as exc: + logger.error("LAiSER API poll returned non-JSON (jobId=%s): %s", job_id, exc) + return {"error": f"Invalid JSON from LAiSER poll: {exc}"} + + if data.get("status") in ("QUEUED", "RUNNING"): + continue + + return data + + logger.error("LAiSER API timed out after %ds, jobId=%s", timeout, job_id) + return {"error": f"LAiSER API timed out after {timeout} seconds"} + + @staticmethod + def _normalize_laiser_skill(skill: dict) -> dict: + """Map a LAiSER API result entry to the internal skills alignment format.""" + source = (skill.get("Taxonomy Source") or "").lower() + target_type_map = { + "esco": "ESCO:Skill", + "ukos": "UKOS:Skill", + "onet_tech": "ONET:Skill", + } + return { + "type": "Alignment", + "skill_tag": skill.get("Raw Skill", ""), + "target_name": skill.get("Taxonomy Skill", ""), + "target_description": skill.get("Taxonomy Description", ""), + "target_url": skill.get("Source URL", ""), + "target_type": target_type_map.get(source, source), + "correlation_coefficient": skill.get("Correlation Coefficient", 0), + "task_abilities": [], + "knowledge_required": [], + } diff --git a/backend/openedx_ai_badges/settings/common.py b/backend/openedx_ai_badges/settings/common.py index 86bb80f..ce3e7e8 100644 --- a/backend/openedx_ai_badges/settings/common.py +++ b/backend/openedx_ai_badges/settings/common.py @@ -37,3 +37,11 @@ def plugin_settings(settings): settings.MIT_SLM_OLLAMA_URL = "" settings.MIT_SLM_OLLAMA_TOKEN = "" settings.MIT_DCC_BADGE_API_HEALTH_URL = "http://mit-slm:8000/health" + + # ------------------------- + # LAiSER API + # ------------------------- + settings.LAISER_API_BASE_URL = "" + settings.LAISER_API_KEY = "" + settings.LAISER_API_TIMEOUT_SECONDS = 90 + settings.LAISER_API_POLL_INTERVAL_SECONDS = 2 diff --git a/backend/openedx_ai_badges/settings/production.py b/backend/openedx_ai_badges/settings/production.py index d4c4ea8..7395b7f 100644 --- a/backend/openedx_ai_badges/settings/production.py +++ b/backend/openedx_ai_badges/settings/production.py @@ -31,3 +31,20 @@ def plugin_settings(settings): settings.MIT_DCC_BADGE_API_HEALTH_URL = settings.ENV_TOKENS.get( "MIT_DCC_BADGE_API_HEALTH_URL", settings.MIT_DCC_BADGE_API_HEALTH_URL ) + + # ------------------------- + # LAiSER API + # ------------------------- + if hasattr(settings, "ENV_TOKENS"): + settings.LAISER_API_BASE_URL = settings.ENV_TOKENS.get( + "LAISER_API_BASE_URL", settings.LAISER_API_BASE_URL + ) + settings.LAISER_API_KEY = settings.ENV_TOKENS.get( + "LAISER_API_KEY", settings.LAISER_API_KEY + ) + settings.LAISER_API_TIMEOUT_SECONDS = settings.ENV_TOKENS.get( + "LAISER_API_TIMEOUT_SECONDS", settings.LAISER_API_TIMEOUT_SECONDS + ) + settings.LAISER_API_POLL_INTERVAL_SECONDS = settings.ENV_TOKENS.get( + "LAISER_API_POLL_INTERVAL_SECONDS", settings.LAISER_API_POLL_INTERVAL_SECONDS + ) diff --git a/backend/openedx_ai_badges/workflows/orchestrators.py b/backend/openedx_ai_badges/workflows/orchestrators.py index af14b1b..b49359a 100644 --- a/backend/openedx_ai_badges/workflows/orchestrators.py +++ b/backend/openedx_ai_badges/workflows/orchestrators.py @@ -234,7 +234,8 @@ def regenerate(self, input_data): } if skills_requested: - self._set_status_message("Generating skills alignment...") + skills_fn = self.profile.processor_config.get("SkillsProcessor", {}).get("function", "generate_skills") + self._set_status_message(f'Generating skills alignment using "{skills_fn}"...') skills = self._get_skills(course_context, input_data, regenerate=True) if isinstance(skills, dict) and 'error' in skills: return skills @@ -442,7 +443,8 @@ def run(self, input_data): } if skills_enabled: - self._set_status_message("Generating skills alignment...") + skills_fn = self.profile.processor_config.get("SkillsProcessor", {}).get("function", "generate_skills") + self._set_status_message(f'Generating skills alignment using "{skills_fn}"...') skills = self._get_skills(course_context, input_data) if isinstance(skills, dict) and 'error' in skills: return skills diff --git a/backend/openedx_ai_badges/workflows/profiles/badges_base.json b/backend/openedx_ai_badges/workflows/profiles/badges_base.json index d1396b9..4c6278b 100644 --- a/backend/openedx_ai_badges/workflows/profiles/badges_base.json +++ b/backend/openedx_ai_badges/workflows/profiles/badges_base.json @@ -10,8 +10,9 @@ "provider": "openai", }, "SkillsProcessor": { - "function": "generate_skills", - "provider": "openai", + "function": "generate_skills_laiser_api", + // "function": "generate_skills", + // "provider": "openai", } }, "actuator_config": { diff --git a/tutor/openedx_ai_badges/patches/openedx-common-settings b/tutor/openedx_ai_badges/patches/openedx-common-settings index cb37437..dcb46ca 100644 --- a/tutor/openedx_ai_badges/patches/openedx-common-settings +++ b/tutor/openedx_ai_badges/patches/openedx-common-settings @@ -1,3 +1,5 @@ +LAISER_API_BASE_URL = "{{ LAISER_API_BASE_URL }}" +LAISER_API_KEY = "{{ LAISER_API_KEY }}" MIT_SLM_OLLAMA_URL = "{{ MIT_SLM_OLLAMA_URL }}" MIT_SLM_OLLAMA_TOKEN = "{{ MIT_SLM_OLLAMA_TOKEN }}" MIT_DCC_BADGE_API_HEALTH_URL = "{{ MIT_DCC_BADGE_API_HEALTH_URL }}" diff --git a/tutor/openedx_ai_badges/plugin.py b/tutor/openedx_ai_badges/plugin.py index a287e66..41f8ea8 100644 --- a/tutor/openedx_ai_badges/plugin.py +++ b/tutor/openedx_ai_badges/plugin.py @@ -53,6 +53,8 @@ def _mount_plugin(mounts, path): hooks.Filters.CONFIG_DEFAULTS.add_items( [ + ("LAISER_API_BASE_URL", ""), + ("LAISER_API_KEY", ""), ("RUN_MIT_SLM", False), ("MIT_SLM_DOCKER_IMAGE", "felipemontoya/dcc-mit-badge-api:latest"), ("MIT_SLM_OLLAMA_URL", "https://felipemontoya-mit-dcc-ollama.hf.space/api/generate"), From c2996961b48fde2adbd157e1dc3eb34b63a69d5f Mon Sep 17 00:00:00 2001 From: Felipe Montoya Date: Tue, 7 Apr 2026 23:01:16 -0500 Subject: [PATCH 2/2] feat: adding a draft version of laiser locally --- backend/Makefile | 3 + .../processors/badge_processor.py | 86 +++++++++++++++++++ backend/openedx_ai_badges/settings/common.py | 6 ++ .../openedx_ai_badges/settings/production.py | 11 +++ backend/requirements/laiser.in | 6 ++ 5 files changed, 112 insertions(+) create mode 100644 backend/requirements/laiser.in diff --git a/backend/Makefile b/backend/Makefile index d17b54a..880aa4b 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -70,6 +70,9 @@ requirements: clean_tox piptools ## install development environment requirements # So that the plugin entrypoints are installed and loaded correctly. pip install -e . +requirements-laiser: ## install optional LAiSER local extractor dependencies (CPU mode) + pip install -r requirements/laiser.in + test: clean ## run tests in the current virtualenv pytest diff --git a/backend/openedx_ai_badges/processors/badge_processor.py b/backend/openedx_ai_badges/processors/badge_processor.py index 29c0fdb..f526f79 100644 --- a/backend/openedx_ai_badges/processors/badge_processor.py +++ b/backend/openedx_ai_badges/processors/badge_processor.py @@ -4,6 +4,7 @@ import json import logging import time +from functools import lru_cache from pathlib import Path import requests @@ -13,6 +14,13 @@ logger = logging.getLogger(__name__) +@lru_cache(maxsize=4) +def _get_laiser_extractor(model_id, hf_token, use_gpu): + """Return a cached Skill_Extractor instance keyed by (model_id, hf_token, use_gpu).""" + from laiser.skill_extractor import Skill_Extractor # pylint: disable=import-error,import-outside-toplevel + return Skill_Extractor(AI_MODEL_ID=model_id, HF_TOKEN=hf_token, use_gpu=use_gpu) + + class BaseBadgeLLMProcessor(LLMProcessor): """Base processor for badge-related LLM tasks.""" schema_filename = None @@ -100,6 +108,58 @@ def generate_skills(self): prompt = self.fill_prompt(prompt) result = self._call_completion_wrapper(system_role=prompt) return result + + def generate_skills_laiser_local(self): + """ + Extract skills from course context using the LAiSER library running in-process. + + Initializes Skill_Extractor once per (model_id, hf_token, use_gpu) combination + and caches it for subsequent calls. Wraps the context in a DataFrame, runs + extraction, enriches results with ESCO metadata, and normalizes to the internal + skills alignment format. + """ + try: + import pandas as pd # pylint: disable=import-outside-toplevel + except ImportError as exc: + logger.error("LAiSER local: pandas not installed: %s", exc) + return {"error": f"pandas not available: {exc}"} + + model_id = self.config.get("model_id") or getattr(settings, "LAISER_MODEL_ID", "") + hf_token = self.config.get("hf_token") or getattr(settings, "LAISER_HF_TOKEN", "") + use_gpu = self.config.get("use_gpu", False) + top_k = self.config.get("top_k", 10) + + if not model_id: + logger.error("LAiSER local: LAISER_MODEL_ID is not configured") + return {"error": "LAISER_MODEL_ID is not configured"} + + try: + extractor = _get_laiser_extractor(model_id, hf_token, use_gpu) + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error("LAiSER local: failed to initialize extractor: %s", exc) + return {"error": f"LAiSER extractor initialization failed: {exc}"} + + data = pd.DataFrame({"id": ["badge_1"], "description": [self.context or ""]}) + + try: + result_df = extractor.extractor( + data=data, + id_column="id", + text_columns=["description"], + input_type="job_desc", + top_k=top_k, + levels=False, + warnings=False, + ) + except Exception as exc: # pylint: disable=broad-exception-caught + logger.error("LAiSER local: extraction failed: %s", exc) + return {"error": f"LAiSER extraction failed: {exc}"} + + raw_skills = result_df.to_dict("records") if isinstance(result_df, pd.DataFrame) else [] + + skills = [self._normalize_laiser_local_skill(s, extractor) for s in raw_skills] + return {"response": json.dumps({"skills": skills}), "status": "success"} + def generate_skills_laiser_api(self): """ Submit course context to the LAiSER API and poll for extracted skills. @@ -198,3 +258,29 @@ def _normalize_laiser_skill(skill: dict) -> dict: "task_abilities": [], "knowledge_required": [], } + + @staticmethod + def _normalize_laiser_local_skill(skill: dict, extractor) -> dict: + """Map a LAiSER local extractor result row to the internal skills alignment format.""" + raw_skill = skill.get("Raw Skill", "") + + target_description = "" + target_url = "" + if raw_skill and getattr(extractor, "esco_df", None) is not None: + match = extractor.esco_df[extractor.esco_df["preferredLabel"] == raw_skill] + if not match.empty: + row = match.iloc[0] + target_description = row.get("description", "") + target_url = row.get("conceptUri", "") + + return { + "type": "Alignment", + "skill_tag": skill.get("Skill Tag", ""), + "target_name": raw_skill, + "target_description": target_description, + "target_url": target_url, + "target_type": "ESCO:Skill", + "correlation_coefficient": skill.get("Correlation Coefficient", 0), + "task_abilities": skill.get("Task Abilities", []), + "knowledge_required": skill.get("Knowledge Required", []), + } diff --git a/backend/openedx_ai_badges/settings/common.py b/backend/openedx_ai_badges/settings/common.py index ce3e7e8..3589792 100644 --- a/backend/openedx_ai_badges/settings/common.py +++ b/backend/openedx_ai_badges/settings/common.py @@ -45,3 +45,9 @@ def plugin_settings(settings): settings.LAISER_API_KEY = "" settings.LAISER_API_TIMEOUT_SECONDS = 90 settings.LAISER_API_POLL_INTERVAL_SECONDS = 2 + + # ------------------------- + # LAiSER local + # ------------------------- + settings.LAISER_MODEL_ID = "" + settings.LAISER_HF_TOKEN = "" diff --git a/backend/openedx_ai_badges/settings/production.py b/backend/openedx_ai_badges/settings/production.py index 7395b7f..2671342 100644 --- a/backend/openedx_ai_badges/settings/production.py +++ b/backend/openedx_ai_badges/settings/production.py @@ -48,3 +48,14 @@ def plugin_settings(settings): settings.LAISER_API_POLL_INTERVAL_SECONDS = settings.ENV_TOKENS.get( "LAISER_API_POLL_INTERVAL_SECONDS", settings.LAISER_API_POLL_INTERVAL_SECONDS ) + + # ------------------------- + # LAiSER local + # ------------------------- + if hasattr(settings, "ENV_TOKENS"): + settings.LAISER_MODEL_ID = settings.ENV_TOKENS.get( + "LAISER_MODEL_ID", settings.LAISER_MODEL_ID + ) + settings.LAISER_HF_TOKEN = settings.ENV_TOKENS.get( + "LAISER_HF_TOKEN", settings.LAISER_HF_TOKEN + ) diff --git a/backend/requirements/laiser.in b/backend/requirements/laiser.in new file mode 100644 index 0000000..64f462e --- /dev/null +++ b/backend/requirements/laiser.in @@ -0,0 +1,6 @@ +# Optional requirements for running the LAiSER local skill extractor. +# Install with: pip install -r requirements/laiser.txt +# or: make requirements-laiser + +laiser[cpu] # LAiSER local skill extraction (CPU mode) +pandas # Required by the LAiSER extractor