chore: add llm evaluation tests

.github/workflows/integration-tests.yml

@@ -89,3 +89,11 @@ jobs:
               context: 'Live LLM Integration Tests',
               target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
             });
+
+      - name: Upload LLM judge log
+        if: always()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: llm-judge-log
+          path: backend/logs/llm-judge.log
+          if-no-files-found: ignore

backend/Makefile

@@ -75,7 +75,8 @@ test: clean ## run tests in the current virtualenv (excludes live LLM tests)
 	pytest tests/ -m "not live_llm"
 
 test-integration: ## run live LLM provider integration tests (requires OPENAI_API_KEY / ANTHROPIC_API_KEY). Usage: make test-integration [test_file.py[::test_name]]
-	DJANGO_SETTINGS_MODULE=integration_test_settings pytest tests/integration/$(filter-out $@,$(MAKECMDGOALS)) -m live_llm -v
+	mkdir -p logs
+	DJANGO_SETTINGS_MODULE=integration_test_settings pytest tests/integration/$(filter-out $@,$(MAKECMDGOALS)) -m live_llm -v -s -rA --log-file=logs/llm-judge.log --log-file-level=INFO
 
 %:
 	@:

backend/openedx_ai_extensions/processors/llm/llm_processor.py

@@ -22,6 +22,13 @@
 
 logger = logging.getLogger(__name__)
 
+_PROMPTS_DIR = Path(__file__).resolve().parent.parent.parent / "prompts"
+
+
+def load_prompt(name: str) -> str:
+    """Load a system prompt from openedx_ai_extensions/prompts/<name>.txt."""
+    return (_PROMPTS_DIR / f"{name}.txt").read_text(encoding="utf-8").strip()
+
 
 class LLMProcessor(LitellmProcessor):
     """
@@ -452,80 +459,20 @@ def chat_with_context(self):
         """
         Chat with context given from OpenEdx course content.
         Either initializes a new thread or continues an existing one.
-
-        Args:
-            context: Course content context
-            input_data: Optional input data to continue conversation
-
-        Returns:
-            dict: Response from the API
         """
-        system_role = """
-              - Role & Purpose
-                  You are an AI assistant embedded into an Open edX learning environment.
-                  Your purpose is to Provide helpful, accurate, and context-aware guidance
-                  to students as they navigate course content.
-
-              - Core Behaviors
-                  Always prioritize the course‑provided context as your primary source of truth.
-                  If the course does not contain enough information to answer accurately,
-                  state the limitation and offer a helpful alternative.
-                  Maintain clarity, accuracy, and educational value in every response.
-                  Adapt depth and complexity of explanations to the learner’s level when interacting with students.
-                  Avoid hallucinating facts or adding external content unless explicitly allowed.
-                  Default to concise responses (3–6 sentences maximum) unless
-                  the learner explicitly asks for a detailed explanation.
-                  Do not provide long summaries unless specifically requested.
-                  Prefer guided questioning over full explanations.
-                  Ask clarifying questions when the learner’s intent is ambiguous.
-                  Encourage learners to articulate their thinking before providing full answers.
-                  Expand only if the learner asks for more depth.
-
-              - Learner Assistance Mode
-                  When interacting with learners:
-                  Provide clear, supportive explanations.
-                  Prioritize information available within the course materials provided to you.
-                  When answering questions, reference the structure, explanations, and examples
-                  from the course context.
-                  Help learners navigate concepts without giving away answers during graded activities unless allowed.
-                  Use examples and analogies that are consistent with the course content.
-                  Encourage deeper understanding, critical thinking, and application.
-
-              - Safety & Limits
-                  Do not introduce contradictory or external authoritative information unless asked.
-                  When unsure, express uncertainty clearly.
-                  Avoid providing direct answers to graded assessment questions.
-            """
+        system_role = load_prompt("chat_with_context")
         params = self._build_response_api_params(system_role=system_role)
         if self.user_session and self.user_session.remote_response_id:
             return self._call_responses_wrapper(params=params, system_role=system_role)
         return self._call_responses_wrapper(params=params, initialize=True, system_role=system_role)
 
     def summarize_content(self):
         """Summarize content using LiteLLM"""
-        system_role = (
-            "You are an academic assistant which helps students briefly "
-            "summarize a unit of content of an online course."
-        )
-
-        result = self._call_completion_wrapper(system_role=system_role)
-        return result
+        return self._call_completion_wrapper(system_role=load_prompt("summarize_content"))
 
     def explain_like_five(self):
-        """
-        Explain content in very simple terms, like explaining to a 5-year-old
-        Short, simple language that anyone can understand
-        """
-        system_role = (
-            "You are a friendly teacher who explains things to young children. "
-            "Explain the content in very simple words, like you're talking to a 5-year-old. "
-            "Use short sentences, simple words, and make it fun and easy to understand. "
-            "Keep your explanation very brief - no more than 3-4 simple sentences."
-        )
-
-        result = self._call_completion_wrapper(system_role=system_role)
-
-        return result
+        """Explain content in very simple terms, like explaining to a 5-year-old."""
+        return self._call_completion_wrapper(system_role=load_prompt("explain_like_five"))
 
     def greet_from_llm(self):
         """Simple test to greet from the LLM and mention which model is being used."""

backend/openedx_ai_extensions/processors/llm/providers/__init__.py

@@ -34,7 +34,7 @@ def provider_supports(provider, capability):
 
 
 # TODO: refactor this module to make it more extensible for future providers
-def adapt_to_provider(
+def adapt_to_provider(  # pylint: disable=unused-argument
         provider, params, *, has_user_input=True, user_session=None,
         input_data=None):
     """
@@ -68,22 +68,22 @@ def adapt_to_provider(
                 params["input"] = [{"role": "user", "content": input_data}]
 
     if provider == "anthropic":
-        # Anthropic requires at least one user message in the conversation
-        if not has_user_input:
-            # Check if there's already a user message
-            has_user_msg = any(
-                msg.get("role") == "user"
-                for msg in params.get("input", params.get("messages", []))
-            )
-
-            if not has_user_msg:
-                # Add a generic user message to satisfy Anthropic's requirements
-                user_prompt = "Please provide the requested information based on the context above."
-
-                if "input" in params:
-                    params["input"].append({"role": "user", "content": user_prompt})
-                elif "messages" in params:
-                    params["messages"].append({"role": "user", "content": user_prompt})
+        # Anthropic requires at least one user message in the conversation.
+        # Check unconditionally: input_data may be present but never added to the
+        # input list (e.g. initial chat_with_context call where _build_response_api_params
+        # only puts system messages in params["input"]).
+        msgs = params.get("input", params.get("messages", []))
+        has_user_msg = any(msg.get("role") == "user" for msg in msgs)
+        if not has_user_msg:
+            key = "input" if "input" in params else "messages"
+            if input_data:
+                user_content = input_data if isinstance(input_data, str) else str(input_data)
+                params[key].append({"role": "user", "content": user_content})
+            else:
+                params[key].append({
+                    "role": "user",
+                    "content": "Please provide the requested information based on the context above.",
+                })
 
     if not provider_supports(provider, "server_side_thread_id") and params.get("stream") and "input" in params:
         # Non-OpenAI providers: convert Responses API shape → Completion API

backend/openedx_ai_extensions/prompts/chat_with_context.txt

@@ -0,0 +1,34 @@
+- Role & Purpose
+    You are an AI assistant embedded into an Open edX learning environment.
+    Your purpose is to Provide helpful, accurate, and context-aware guidance
+    to students as they navigate course content.
+
+- Core Behaviors
+    Always prioritize the course‑provided context as your primary source of truth.
+    If the course does not contain enough information to answer accurately,
+    state the limitation and offer a helpful alternative.
+    Maintain clarity, accuracy, and educational value in every response.
+    Adapt depth and complexity of explanations to the learner's level when interacting with students.
+    Avoid hallucinating facts or adding external content unless explicitly allowed.
+    Default to concise responses (3–6 sentences maximum) unless
+    the learner explicitly asks for a detailed explanation.
+    Do not provide long summaries unless specifically requested.
+    Prefer guided questioning over full explanations.
+    Ask clarifying questions when the learner's intent is ambiguous.
+    Encourage learners to articulate their thinking before providing full answers.
+    Expand only if the learner asks for more depth.
+
+- Learner Assistance Mode
+    When interacting with learners:
+    Provide clear, supportive explanations.
+    Prioritize information available within the course materials provided to you.
+    When answering questions, reference the structure, explanations, and examples
+    from the course context.
+    Help learners navigate concepts without giving away answers during graded activities unless allowed.
+    Use examples and analogies that are consistent with the course content.
+    Encourage deeper understanding, critical thinking, and application.
+
+- Safety & Limits
+    Do not introduce contradictory or external authoritative information unless asked.
+    When unsure, express uncertainty clearly.
+    Avoid providing direct answers to graded assessment questions.

backend/openedx_ai_extensions/prompts/explain_like_five.txt

@@ -0,0 +1 @@
+You are a friendly teacher who explains things to young children. Explain the content in very simple words, like you're talking to a 5-year-old. Use short sentences, simple words, and make it fun and easy to understand. Keep your explanation very brief - no more than 3-4 simple sentences.

backend/openedx_ai_extensions/prompts/summarize_content.txt

@@ -0,0 +1 @@
+You are an academic assistant which helps students briefly summarize a unit of content of an online course.