Add simple LLM examples

docs/source/llm_examples/async_concurrency.py

@@ -0,0 +1,61 @@
+"""Fork/join async concurrency with templates.
+
+Demonstrates:
+- Running multiple LLM template calls concurrently with ``asyncio.gather``
+- Using ``asyncio.to_thread`` to run synchronous template calls in parallel
+"""
+
+import argparse
+import asyncio
+import functools
+import os
+
+from effectful.handlers.llm import Template
+from effectful.handlers.llm.completions import LiteLLMProvider
+from effectful.ops.semantics import handler
+from effectful.ops.types import NotHandled
+
+# ---------------------------------------------------------------------------
+# Async template
+# ---------------------------------------------------------------------------
+
+
+@Template.define
+def analyze_average_age(ages: list[int]) -> int:
+    """Analyze the dataset of ages {ages} and return the average age of
+    participants. Do not use any tools."""
+    raise NotHandled
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+async def main(provider: LiteLLMProvider):
+    analysis = functools.partial(
+        asyncio.to_thread, handler(provider)(analyze_average_age)
+    )
+    results = await asyncio.gather(
+        analysis([25, 30, 35, 40]),
+        analysis([20, 28, 17, 30]),
+        analysis([22, 27, 31, 29]),
+        analysis([24, 26, 32, 38]),
+        analysis([21, 29, 33, 37]),
+    )
+    for i, result in enumerate(results):
+        print(f"Group {i}: average age = {result}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Analyze average ages concurrently")
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=os.environ.get("EFFECTFUL_LLM_MODEL", ""),
+        help="LLM model to use",
+    )
+    args = parser.parse_args()
+
+    provider = LiteLLMProvider(model=args.model)
+    asyncio.run(main(provider))

docs/source/llm_examples/batch_translate.py

@@ -0,0 +1,70 @@
+"""Batch translation with instruction injection.
+
+Demonstrates:
+- ``@Template.define`` for a translation template with injected instructions
+"""
+
+import argparse
+import os
+
+from tenacity import stop_after_attempt
+
+from effectful.handlers.llm import Template
+from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler
+from effectful.handlers.llm.evaluation import RestrictedEvalProvider
+from effectful.ops.semantics import handler
+from effectful.ops.types import NotHandled
+
+# ---------------------------------------------------------------------------
+# Translation template
+# ---------------------------------------------------------------------------
+
+
+@Template.define
+def translate(target_language: str, instructions: str = "") -> Template[[str], str]:
+    """
+    Write a `Template` that translates a string of English text into {target_language}
+    If any instructions are provided, include them in the prompt: {instructions}
+    """
+    raise NotHandled
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Batch translation with instruction injection"
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=os.environ.get("EFFECTFUL_LLM_MODEL", ""),
+        help="LLM model to use",
+    )
+    parser.add_argument(
+        "--max-steps",
+        type=int,
+        default=5,
+        help="Maximum number of steps before giving up",
+    )
+    parser.add_argument(
+        "--num-retries",
+        type=int,
+        default=5,
+        help="Number of retries for malformed LLM output",
+    )
+    args = parser.parse_args()
+
+    provider = LiteLLMProvider(model=args.model)
+
+    with (
+        handler(provider),
+        handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))),
+        handler(RestrictedEvalProvider()),
+    ):
+        translator = translate(
+            target_language="french", instructions="Use formal language."
+        )
+        print(translator("hello, how are you? how is your day going?"))

docs/source/llm_examples/chat_memory.py

@@ -0,0 +1,124 @@
+"""Chat agent with embedding-based memory.
+
+Demonstrates:
+- A stateful chat agent that maintains conversation history
+- Embedding-based retrieval of relevant past context
+- Simple in-memory vector store with L2 distance
+"""
+
+import argparse
+import dataclasses
+import os
+
+import litellm
+import numpy as np
+
+from effectful.handlers.llm import Template
+from effectful.handlers.llm.completions import LiteLLMProvider
+from effectful.ops.semantics import handler
+from effectful.ops.types import NotHandled
+
+# ---------------------------------------------------------------------------
+# Embedding helpers
+# ---------------------------------------------------------------------------
+
+
+def get_embedding(text: str) -> np.ndarray:
+    """Get an embedding vector for the given text using litellm."""
+    response = litellm.embedding(model="text-embedding-ada-002", input=text)
+    return np.array(response.data[0]["embedding"], dtype=np.float32)
+
+
+def find_closest(
+    index: list[tuple[str, np.ndarray]], phrase: str
+) -> tuple[str, float] | None:
+    """Find the closest entry in the index to the given phrase."""
+    if not index:
+        return None
+    phrase_embedding = get_embedding(phrase)
+
+    def dist(a: np.ndarray, b: np.ndarray) -> float:
+        return float(((a - b) ** 2).sum())
+
+    return min(
+        ((msg, dist(embedding, phrase_embedding)) for msg, embedding in index),
+        key=lambda elt: elt[1],
+    )
+
+
+# ---------------------------------------------------------------------------
+# Chat template
+# ---------------------------------------------------------------------------
+
+
+@Template.define
+def respond_to_user(
+    user_message: str, relevant_context: str, prev_messages: str
+) -> str:
+    """Given the user wrote: {user_message}
+    Continue the conversation.
+    The last few messages were: {prev_messages}
+    Older relevant context: {relevant_context}"""
+    raise NotHandled
+
+
+# ---------------------------------------------------------------------------
+# Chat agent
+# ---------------------------------------------------------------------------
+
+
+@dataclasses.dataclass
+class ChatAgent:
+    """A chat agent that compresses old messages into an embedding index."""
+
+    history: list[dict[str, str]] = dataclasses.field(default_factory=list)
+    index: list[tuple[str, np.ndarray]] = dataclasses.field(default_factory=list)
+
+    def _compress(self):
+        """Move the oldest pair of messages into the embedding index."""
+        oldest_pair, self.history = self.history[:2], self.history[2:]
+        text = "\n".join(m["content"] for m in oldest_pair)
+        self.index.append((text, get_embedding(text)))
+
+    def _find_relevant(self, query: str) -> str:
+        result = find_closest(self.index, query)
+        return result[0] if result else "No relevant context."
+
+    def chat(self, user_input: str):
+        relevant = self._find_relevant(user_input)
+        prev_messages = "\n".join(
+            f"{m['author']}: {m['content']}" for m in self.history
+        )
+        response = respond_to_user(user_input, relevant, prev_messages)
+        self.history.append({"author": "user", "content": user_input})
+        self.history.append({"author": "agent", "content": response})
+        if len(self.history) > 6:
+            self._compress()
+        print(f"user: {user_input}")
+        print(f"agent: {response}")
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Chat agent with embedding-based memory"
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=os.environ.get("EFFECTFUL_LLM_MODEL", ""),
+        help="LLM model to use",
+    )
+    args = parser.parse_args()
+
+    agent = ChatAgent()
+
+    provider = LiteLLMProvider(model=args.model)
+    with handler(provider):
+        agent.chat("Hello! How are you doing?")
+        agent.chat("Lovely! I'm having a great day.")
+        agent.chat("What is the capital of France?")
+        agent.chat("I didn't know that! That's amazing!")

docs/source/llm_examples/chat_search.py

@@ -0,0 +1,116 @@
+import argparse
+import dataclasses
+import os
+import urllib.parse
+
+import requests
+from tenacity import stop_after_attempt
+
+from effectful.handlers.llm import Agent, Template, Tool
+from effectful.handlers.llm.completions import LiteLLMProvider, RetryLLMHandler
+from effectful.ops.semantics import handler
+from effectful.ops.types import NotHandled
+
+
+@Tool.define
+def search_web(query: str) -> str:
+    """Search Wikipedia for a topic and return a summary. The query can be a topic name or a natural language question."""
+    search_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode(
+        {
+            "action": "query",
+            "list": "search",
+            "srsearch": query,
+            "srlimit": 1,
+            "format": "json",
+        }
+    )
+    search_data = requests.get(
+        search_url, headers={"User-Agent": "effectful-example/1.0"}
+    ).json()
+    results = search_data.get("query", {}).get("search", [])
+    if not results:
+        raise ValueError(f"No results found for: {query}")
+    title = results[0]["title"]
+
+    summary_url = "https://en.wikipedia.org/w/api.php?" + urllib.parse.urlencode(
+        {
+            "action": "query",
+            "titles": title,
+            "prop": "extracts",
+            "exintro": True,
+            "explaintext": True,
+            "format": "json",
+        }
+    )
+    summary_data = requests.get(
+        summary_url, headers={"User-Agent": "effectful-example/1.0"}
+    ).json()
+    page = next(iter(summary_data["query"]["pages"].values()))
+    extract = page.get("extract", "No summary available.")
+    url = f"https://en.wikipedia.org/wiki/{urllib.parse.quote(title.replace(' ', '_'))}"
+
+    return f"# {title}\n\n{extract}\n\nSource: {url}"
+
+
+@dataclasses.dataclass
+class ChatBot(Agent):
+    """Simple chat agent for testing history accumulation."""
+
+    bot_name: str = dataclasses.field(default="ChatBot")
+
+    @Template.define
+    def send(self, user_input: str) -> str:
+        """
+        You are a friendly and helpful AI assistant named {self.bot_name}.
+        If user input contains a question that you're not sure how to answer,
+        consider using the web search tool to find the answer and include it in your response.
+
+        The user writes:
+        {user_input}
+        """
+        raise NotHandled
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="LLM-guided research agent with web search"
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=os.environ.get("EFFECTFUL_LLM_MODEL", ""),
+        help="LLM model to use",
+    )
+    parser.add_argument(
+        "--name",
+        type=str,
+        default="Chatty McChatface",
+        help="The name of the chatbot",
+    )
+    parser.add_argument(
+        "--interactive",
+        action="store_true",
+        help="Run in interactive mode, allowing multiple back-and-forth messages",
+    )
+    parser.add_argument(
+        "--num-retries",
+        type=int,
+        default=4,
+        help="Number of retries for malformed LLM output",
+    )
+    args = parser.parse_args()
+
+    chatbot = ChatBot(bot_name=args.name)
+    provider = LiteLLMProvider(model=args.model)
+
+    with (
+        handler(provider),
+        handler(RetryLLMHandler(stop=stop_after_attempt(args.num_retries))),
+    ):
+        if args.interactive:
+            while True:
+                print(chatbot.send(input("You: ")))
+        else:
+            print(chatbot.send("Hi! Can you tell me about the Statue of Liberty?"))
+            print(chatbot.send("Who designed it?"))
+            print(chatbot.send("What about the speed of light? How fast is it?"))