Merge pull request #236 from SpudGunMan/copilot/link-llm-to-wiki-module

Add RAG support to LLM module with Wikipedia/Kiwix and OpenWebUI integration

README.md

@@ -42,7 +42,7 @@ Mesh Bot is a feature-rich Python bot designed to enhance your [Meshtastic](http
 ### Interactive AI and Data Lookup
 - **Weather, Earthquake, River, and Tide Data**: Get local alerts and info from NOAA/USGS; uses Open-Meteo for areas outside NOAA coverage.
 - **Wikipedia Search**: Retrieve summaries from Wikipedia.
-- **Ollama LLM Integration**: Query the [Ollama](https://github.com/ollama/ollama/tree/main/docs) AI for advanced responses.
+- **Ollama LLM Integration**: Query the [Ollama](https://github.com/ollama/ollama/tree/main/docs) AI for advanced responses. Supports RAG (Retrieval Augmented Generation) with Wikipedia/Kiwix context and [OpenWebUI](https://github.com/open-webui/open-webui) integration for enhanced AI capabilities.
 - **Satellite Passes**: Find upcoming satellite passes for your location.
 - **GeoMeasuring Tools**: Calculate distances and midpoints using collected GPS data; supports Fox & Hound direction finding.
 

config.template

@@ -75,15 +75,28 @@ kiwixLibraryName = wikipedia_en_100_nopic_2025-09
 
 # Enable ollama LLM see more at https://ollama.com
 ollama = False
-# Ollama model to use (defaults to gemma3:270m)
+# Ollama model to use (defaults to gemma3:270m) gemma2 is good for older SYSTEM prompt
 # ollamaModel = gemma3:latest
+# ollamaModel = gemma2:2b
 # server instance to use (defaults to local machine install)
 ollamaHostName = http://localhost:11434
+
 # Produce LLM replies to messages that aren't commands?
 # If False, the LLM only replies to the "ask:" and "askai" commands.
 llmReplyToNonCommands = True
-# if True, the input is sent raw to the LLM, if False uses legacy template query
-rawLLMQuery = True 
+# if True, the input is sent raw to the LLM, if False uses SYSTEM prompt
+rawLLMQuery = True
+
+# Enable Wikipedia/Kiwix integration with LLM for RAG (Retrieval Augmented Generation)
+# When enabled, LLM will automatically search Wikipedia/Kiwix and include context in responses
+llmUseWikiContext = False
+
+# Use OpenWebUI instead of direct Ollama API (enables advanced RAG features)
+useOpenWebUI = False
+# OpenWebUI server URL (e.g., http://localhost:3000)
+openWebUIURL = http://localhost:3000
+# OpenWebUI API key/token (required when useOpenWebUI is True)
+openWebUIAPIKey = 
 
 # StoreForward Enabled and Limits
 StoreForward = True

mesh_bot.py

@@ -1491,10 +1491,21 @@ def handle_boot(mesh=True):
                                 f"{get_name_from_number(myNodeNum, 'short', i)}. NodeID: {myNodeNum}, {decimal_to_hex(myNodeNum)}")
                     
             if llm_enabled:
-                logger.debug(f"System: Ollama LLM Enabled, loading model {my_settings.llmModel} please wait")
-                llmLoad = llm_query(" ")
+                msg = f"System: LLM Enabled"
+                llmLoad = llm_query(" ", init=True)
                 if "trouble" not in llmLoad:
-                    logger.debug(f"System: LLM Model {my_settings.llmModel} loaded")
+                    if my_settings.llmReplyToNonCommands:
+                        msg += " | Reply to DM's Enabled"
+                    if my_settings.llmUseWikiContext:
+                        wiki_source = "Kiwixpedia" if my_settings.use_kiwix_server else "Wikipedia"
+                        msg += f" | {wiki_source} Context Enabled"
+                    if my_settings.useOpenWebUI:
+                        msg += " | OpenWebUI API Enabled"
+                    else:
+                        msg += f" | Ollama API Model {my_settings.llmModel} loaded. Use {'RAW' if my_settings.rawLLMQuery else 'SYSTEM'} prompt mode."
+                    logger.debug(msg)
+                else:
+                    logger.debug(f"System: Bad response from LLM: {llmLoad}")
 
             if my_settings.bbs_enabled:
                 logger.debug(f"System: BBS Enabled, {bbsdb} has {len(bbs_messages)} messages. Direct Mail Messages waiting: {(len(bbs_dm) - 1)}")

modules/README.md

@@ -808,7 +808,7 @@ To set up a local Kiwix server:
 1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y`
 2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim`
 3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim`
-4. Set `useKiwixServer = True` in your config.ini
+4. Set `useKiwixServer = True` in your config.ini with `wikipedia = True`
 
 The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters).
 

modules/llm.md

@@ -0,0 +1,52 @@
+# How do I use this thing?
+This is not a full turnkey setup for Docker yet?
+
+
+# Ollama local
+```bash
+# bash
+curl -fsSL https://ollama.com/install.sh | sh
+# docker
+docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -e OLLAMA_API_BASE_URL=http://host.docker.internal:11434 open-webui/open-webui
+```
+
+```ini
+#service file addition
+# https://github.com/ollama/ollama/issues/703
+[Service]
+Environment="OLLAMA_HOST=0.0.0.0:11434"
+```
+## validation
+http://IP::11434
+`Ollama is running`
+
+# OpenWebUI (docker)
+```bash
+## ollama in docker
+docker run -d -p 3000:8080 --gpus all -v open-webui:/app/backend/data --name open-webui ghcr.io/open-webui/open-webui:cuda
+
+## external ollama
+docker run -d -p 3000:8080 -e OLLAMA_BASE_URL=https://IP:11434 -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main
+```
+wait for engine to build, update the config.ini for the bot
+
+```ini
+# Use OpenWebUI instead of direct Ollama API (enables advanced RAG features)
+useOpenWebUI = True
+# OpenWebUI server URL (e.g., http://localhost:3000)
+openWebUIURL = http://IP:3000
+```
+
+## validation
+http://IP:3000
+make a new admin user.
+validate you have models imported or that the system is working for query.
+make a new user for the bot
+
+
+upper right settings for the user
+settings -> account
+get/create the API key for the user
+
+
+set api endpoint [OpenWebUI API](https://docs.openwebui.com/getting-started/api-endpoints)

modules/llm.py

@@ -3,7 +3,8 @@
 # This module is used to interact with LLM API to generate responses to user input
 # K7MHI Kelly Keeton 2024
 from modules.log import logger
-from modules.settings import llmModel, ollamaHostName, rawLLMQuery
+from modules.settings import (llmModel, ollamaHostName, rawLLMQuery, 
+                              llmUseWikiContext, useOpenWebUI, openWebUIURL, openWebUIAPIKey, cmdBang, urlTimeoutSeconds, use_kiwix_server)
 
 # Ollama Client
 # https://github.com/ollama/ollama/blob/main/docs/faq.md#how-do-i-configure-ollama-server
@@ -11,22 +12,17 @@ import requests
 import json
 from datetime import datetime
 
-if not rawLLMQuery:
-    # this may be removed in the future
-    from googlesearch import search # pip install googlesearch-python
-
 # LLM System Variables
 ollamaAPI = ollamaHostName + "/api/generate"
+openWebUIChatAPI = openWebUIURL + "/api/chat/completions"
+openWebUIOllamaProxy = openWebUIURL + "/ollama/api/generate"
 tokens = 450 # max charcters for the LLM response, this is the max length of the response also in prompts
-requestTruncation = True # if True, the LLM "will" truncate the response 
-
-openaiAPI = "https://api.openai.com/v1/completions" # not used, if you do push a enhancement!
+requestTruncation = True # if True, the LLM "will" truncate the response
+DEBUG_LLM = False # enable debug logging for LLM queries
 
 # Used in the meshBotAI template
 llmEnableHistory = True # enable last message history for the LLM model
-llmContext_fromGoogle = True # enable context from google search results adds to compute time but really helps with responses accuracy
 
-googleSearchResults = 3 # number of google search results to include in the context more results = more compute time
 antiFloodLLM = []
 llmChat_history = {}
 trap_list_llm = ("ask:", "askai")
@@ -52,24 +48,6 @@ meshBotAI = """
 
     """
 
-if llmContext_fromGoogle:
-    meshBotAI = meshBotAI + """
-    CONTEXT
-    The following is the location of the user
-    {location_name}
-
-    The following is for context around the prompt to help guide your response.
-    {context}
-
-    """
-else:
-    meshBotAI = meshBotAI + """
-    CONTEXT
-    The following is the location of the user
-    {location_name}
-
-    """
-
 if llmEnableHistory:
     meshBotAI = meshBotAI + """
     HISTORY
@@ -101,22 +79,6 @@ def llmTool_math_calculator(expression):
     except Exception as e:
         return f"Error in calculation: {e}"
 
-def llmTool_get_google(query, num_results=3):
-    """
-    Example tool function to perform a Google search and return results.
-    :param query: The search query string.
-    :param num_results: Number of search results to return.
-    :return: A list of search result titles and descriptions.
-    """
-    results = []
-    try:
-        googleSearch = search(query, advanced=True, num_results=num_results)
-        for result in googleSearch:
-            results.append(f"{result.title}: {result.description}")
-        return results
-    except Exception as e:
-        return [f"Error in Google search: {e}"]
-
 llmFunctions = [
 
     {
@@ -141,46 +103,164 @@ llmFunctions = [
             "required": ["expression"]
         }
     },
-    {
-        "name": "llmTool_get_google",
-        "description": "Perform a Google search and return results.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "query": {
-                    "type": "string",
-                    "description": "The search query string."
-                },
-                "num_results": {
-                    "type": "integer",
-                    "description": "Number of search results to return.",
-                    "default": 3
-                }
-            },
-            "required": ["query"]
-        }
-    }  
 ]
 
-def get_google_context(input, num_results):
-    # Get context from Google search results
-    googleResults = []
+def get_wiki_context(input):
+    """
+    Get context from Wikipedia/Kiwix for RAG enhancement
+    :param input: The user query
+    :return: Wikipedia summary or empty string if not available
+    """
     try:
-        googleSearch = search(input, advanced=True, num_results=num_results)
-        if googleSearch:
-            for result in googleSearch:
-                googleResults.append(f"{result.title} {result.description}")
-        else:
-            googleResults = ['no other context provided']
+        from modules.wiki import get_wikipedia_summary, get_kiwix_summary
+        # Extract potential search terms from the input
+        # Try to identify key topics/entities for Wikipedia search
+        search_terms = extract_search_terms(input)
+        
+        wiki_context = []
+        for term in search_terms[:2]:  # Limit to 2 searches to avoid excessive API calls
+            if use_kiwix_server:
+                summary = get_kiwix_summary(term, truncate=False)
+            else:
+                summary = get_wikipedia_summary(term, truncate=False)
+
+            if summary and "error" not in summary.lower():
+                wiki_context.append(f"Wikipedia context for '{term}': {summary}")
+        
+        return '\n'.join(wiki_context) if wiki_context else ''
     except Exception as e:
-        logger.debug(f"System: LLM Query: context gathering failed, likely due to network issues")
-        googleResults = ['no other context provided']
-    return googleResults
+        logger.debug(f"System: LLM Query: Wiki context gathering failed: {e}")
+        return ''
+    
+def llm_extract_topic(input):
+    """
+    Use LLM to extract the main topic as a single word or short phrase.
+    Always uses raw mode and supports both Ollama and OpenWebUI.
+    :param input: The user query
+    :return: List with one topic string, or empty list on failure
+    """
+    prompt = (
+        "Summarize the following query into a single word or short phrase that best represents the main topic, "
+        "for use as a Wikipedia search term. Only return the word or phrase, nothing else:\n"
+        f"{input}"
+    )
+    try:
+        if useOpenWebUI and openWebUIAPIKey:
+            result = send_openwebui_query(prompt, max_tokens=10)
+        else:
+            llmQuery = {"model": llmModel, "prompt": prompt, "stream": False, "max_tokens": 10}
+            result = send_ollama_query(llmQuery)
+        topic = result.strip().split('\n')[0]
+        topic = topic.strip(' "\'.,!?;:')
+        if topic:
+            return [topic]
+    except Exception as e:
+        logger.debug(f"LLM topic extraction failed: {e}")
+    return []
+
+def extract_search_terms(input):
+    """
+    Extract potential search terms from user input.
+    Enhanced: Try LLM-based topic extraction first, fallback to heuristic.
+    :param input: The user query
+    :return: List of potential search terms
+    """
+    # Remove common command prefixes
+    for trap in trap_list_llm:
+        if input.lower().startswith(trap):
+            input = input[len(trap):].strip()
+            break
+
+    # Try LLM-based extraction first
+    terms = llm_extract_topic(input)
+    if terms:
+        return terms
+
+    # Fallback: Simple heuristic (existing code)
+    words = input.split()
+    search_terms = []
+    temp_phrase = []
+    for word in words:
+        clean_word = word.strip('.,!?;:')
+        if clean_word and clean_word[0].isupper() and len(clean_word) > 2:
+            temp_phrase.append(clean_word)
+        elif temp_phrase:
+            search_terms.append(' '.join(temp_phrase))
+            temp_phrase = []
+    if temp_phrase:
+        search_terms.append(' '.join(temp_phrase))
+    if not search_terms:
+        search_terms = [input.strip()]
+    if DEBUG_LLM:
+        logger.debug(f"Extracted search terms: {search_terms}")
+    return search_terms[:3]  # Limit to 3 terms
+
+def send_openwebui_query(prompt, model=None, max_tokens=450, context=''):
+    """
+    Send query to OpenWebUI API for chat completion
+    :param prompt: The user prompt
+    :param model: Model name (optional, defaults to llmModel)
+    :param max_tokens: Max tokens for response
+    :param context: Additional context to include
+    :return: Response text or error message
+    """
+    if model is None:
+        model = llmModel
+
+    headers = {
+        'Authorization': f'Bearer {openWebUIAPIKey}',
+        'Content-Type': 'application/json'
+    }
+
+    messages = []
+    if context:
+        messages.append({
+            "role": "system",
+            "content": f"Use the following context to help answer questions:\n{context}"
+        })
+
+    messages.append({
+        "role": "user",
+        "content": prompt
+    })
+
+    data = {
+        "model": model,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "stream": False
+    }
+
+    # Debug logging
+    if DEBUG_LLM:
+        logger.debug(f"OpenWebUI payload: {json.dumps(data)}")
+        logger.debug(f"OpenWebUI endpoint: {openWebUIChatAPI}")
+
+    try:
+        result = requests.post(openWebUIChatAPI, headers=headers, json=data, timeout=urlTimeoutSeconds * 5)
+        if DEBUG_LLM:
+            logger.debug(f"OpenWebUI response status: {result.status_code}")
+            logger.debug(f"OpenWebUI response text: {result.text}")
+        if result.status_code == 200:
+            result_json = result.json()
+            # OpenWebUI returns OpenAI-compatible format
+            if 'choices' in result_json and len(result_json['choices']) > 0:
+                response = result_json['choices'][0]['message']['content']
+                return response.strip()
+            else:
+                logger.warning(f"System: OpenWebUI API returned unexpected format")
+                return "⛔️ Response Error"
+        else:
+            logger.warning(f"System: OpenWebUI API returned status code {result.status_code}")
+            return f"⛔️ Request Error"
+    except requests.exceptions.RequestException as e:
+        logger.warning(f"System: OpenWebUI API request failed: {e}")
+        return f"⛔️ Request Error"
 
 def send_ollama_query(llmQuery):
     # Send the query to the Ollama API and return the response
     try:
-        result = requests.post(ollamaAPI, data=json.dumps(llmQuery), timeout=5)
+        result = requests.post(ollamaAPI, data=json.dumps(llmQuery), timeout= urlTimeoutSeconds * 5)
         if result.status_code == 200:
             result_json = result.json()
             result = result_json.get("response", "")
@@ -219,24 +299,28 @@ def send_ollama_tooling_query(prompt, functions, model=None, max_tokens=450):
     else:
         raise Exception(f"HTTP Error: {result.status_code} - {result.text}")
 
-def llm_query(input, nodeID=0, location_name=None):
+def llm_query(input, nodeID=0, location_name=None, init=False):
     global antiFloodLLM, llmChat_history
-    googleResults = []
+    wikiContext = ''
 
     # if this is the first initialization of the LLM the query of " " should bring meshbotAIinit OTA shouldnt reach this?
     # This is for LLM like gemma and others now?
-    if input == " " and rawLLMQuery:
+    if init and rawLLMQuery:
         logger.warning("System: These LLM models lack a traditional system prompt, they can be verbose and not very helpful be advised.")
         input = meshbotAIinit
-    else:
+    elif init:
         input = input.strip()
         # classic model for gemma2, deepseek-r1, etc
-        logger.debug(f"System: Using classic LLM model framework, ideally for gemma2, deepseek-r1, etc")
+        logger.debug(f"System: Using SYSTEM model framework, ideally for gemma2, deepseek-r1, etc")
 
     if not location_name:
         location_name = "no location provided "
+
+    # Remove command bang if present
+    if cmdBang and input.startswith('!'):
+        input = input.strip('!').strip()
     
-    # remove askai: and ask: from the input
+    # Remove any trap words from the start of the input
     for trap in trap_list_llm:
         if input.lower().startswith(trap):
             input = input[len(trap):].strip()
@@ -251,34 +335,84 @@ def llm_query(input, nodeID=0, location_name=None):
     else:
         antiFloodLLM.append(nodeID)
 
-    if llmContext_fromGoogle and not rawLLMQuery:
-        googleResults = get_google_context(input, googleSearchResults)
+    # Get Wikipedia/Kiwix context if enabled (RAG)
+    if llmUseWikiContext and input != meshbotAIinit:
+        # get_wiki_context returns a string, but we want to count the items before joining
+        search_terms = extract_search_terms(input)
+        wiki_context_list = []
+        for term in search_terms[:2]:
+            if not use_kiwix_server:
+                summary = get_wiki_context(term)
+            else:
+                summary = get_wiki_context(term)
+            if summary and "error" not in summary.lower():
+                wiki_context_list.append(f"Wikipedia context for '{term}': {summary}")
+        wikiContext = '\n'.join(wiki_context_list) if wiki_context_list else ''
+        if wikiContext:
+            logger.debug(f"System: using Wikipedia/Kiwix context for LLM query got {len(wiki_context_list)} results")
 
     history = llmChat_history.get(nodeID, ["", ""])
-
-    if googleResults:
-        logger.debug(f"System: Google-Enhanced LLM Query: {input} From:{nodeID}")
-    else:
-        logger.debug(f"System: LLM Query: {input} From:{nodeID}")
     
     response = ""
     result = ""
     location_name += f" at the current time of {datetime.now().strftime('%Y-%m-%d %H:%M:%S %Z')}"
 
     try:
-        if rawLLMQuery:
-            # sanitize the input to remove tool call syntax
-            if '```' in input:
-                logger.warning("System: LLM Query: Code markdown detected, removing for raw query")
-            input = input.replace('```bash', '').replace('```python', '').replace('```', '')
-            modelPrompt = input
-        else:
-            # Build the query from the template
-            modelPrompt = meshBotAI.format(input=input, context='\n'.join(googleResults), location_name=location_name, llmModel=llmModel, history=history)
+        # Use OpenWebUI if enabled
+        if useOpenWebUI and openWebUIAPIKey:
+            logger.debug(f"System: LLM Query: Using OpenWebUI API for LLM query {input} From:{nodeID}")
             
-        llmQuery = {"model": llmModel, "prompt": modelPrompt, "stream": False, "max_tokens": tokens}
-        # Query the model via Ollama web API
-        result = send_ollama_query(llmQuery)
+            # Combine all context sources
+            combined_context = []
+            if wikiContext:
+                combined_context.append(wikiContext)
+            
+            context_str = '\n\n'.join(combined_context)
+            
+            # For OpenWebUI, we send a cleaner prompt
+            if rawLLMQuery:
+                result = send_openwebui_query(input, context=context_str, max_tokens=tokens)
+            else:
+                # Use the template for non-raw queries
+                modelPrompt = meshBotAI.format(
+                    input=input, 
+                    context=context_str if combined_context else 'no other context provided',
+                    location_name=location_name, 
+                    llmModel=llmModel, 
+                    history=history
+                )
+                result = send_openwebui_query(modelPrompt, max_tokens=tokens)
+        else:
+            logger.debug(f"System: LLM Query: Using Ollama API for LLM query {input} From:{nodeID}")
+            # Use standard Ollama API
+            if rawLLMQuery:
+                # sanitize the input to remove tool call syntax
+                if '```' in input:
+                    logger.warning("System: LLM Query: Code markdown detected, removing for raw query")
+                input = input.replace('```bash', '').replace('```python', '').replace('```', '')
+                modelPrompt = input
+                
+                # Add wiki context to raw queries if available
+                if wikiContext:
+                    modelPrompt = f"Context:\n{wikiContext}\n\nQuestion: {input}"
+            else:
+                # Build the query from the template
+                all_context = []
+                if wikiContext:
+                    all_context.append(wikiContext)
+                
+                context_text = '\n'.join(all_context) if all_context else 'no other context provided'
+                modelPrompt = meshBotAI.format(
+                    input=input, 
+                    context=context_text,
+                    location_name=location_name, 
+                    llmModel=llmModel, 
+                    history=history
+                )
+                
+            llmQuery = {"model": llmModel, "prompt": modelPrompt, "stream": False, "max_tokens": tokens}
+            # Query the model via Ollama web API
+            result = send_ollama_query(llmQuery)
 
         #logger.debug(f"System: LLM Response: " + result.strip().replace('\n', ' '))
     except Exception as e:
@@ -290,13 +424,17 @@ def llm_query(input, nodeID=0, location_name=None):
     response = result.strip().replace('\n', ' ')
     
     if rawLLMQuery and requestTruncation and len(response) > 450:
-        #retryy loop to truncate the response
+        # retry loop to truncate the response
         logger.warning(f"System: LLM Query: Response exceeded {tokens} characters, requesting truncation")
-        truncateQuery = {"model": llmModel, "prompt": truncatePrompt + response, "stream": False, "max_tokens": tokens}
-        truncateResult = send_ollama_query(truncateQuery)
+        truncate_prompt_full = truncatePrompt + response
+        if useOpenWebUI and openWebUIAPIKey:
+            truncateResult = send_openwebui_query(truncate_prompt_full, max_tokens=tokens)
+        else:
+            truncateQuery = {"model": llmModel, "prompt": truncate_prompt_full, "stream": False, "max_tokens": tokens}
+            truncateResult = send_ollama_query(truncateQuery)
 
         # cleanup for message output
-        response = result.strip().replace('\n', ' ')
+        response = truncateResult.strip().replace('\n', ' ')
 
     # done with the query, remove the user from the anti flood list
     antiFloodLLM.remove(nodeID)

modules/settings.py

@@ -256,6 +256,10 @@ try:
     llmModel = config['general'].get('ollamaModel', 'gemma3:270m') # default gemma3:270m
     rawLLMQuery = config['general'].getboolean('rawLLMQuery', True) #default True
     llmReplyToNonCommands = config['general'].getboolean('llmReplyToNonCommands', True) # default True
+    llmUseWikiContext = config['general'].getboolean('llmUseWikiContext', False) # default False
+    useOpenWebUI = config['general'].getboolean('useOpenWebUI', False) # default False
+    openWebUIURL = config['general'].get('openWebUIURL', 'http://localhost:3000') # default localhost:3000
+    openWebUIAPIKey = config['general'].get('openWebUIAPIKey', '') # default empty
     dont_retry_disconnect = config['general'].getboolean('dont_retry_disconnect', False) # default False, retry on disconnect
     favoriteNodeList = config['general'].get('favoriteNodeList', '').split(',')
     enableEcho = config['general'].getboolean('enableEcho', False) # default False

modules/test_bot.py

@@ -97,6 +97,24 @@ class TestBot(unittest.TestCase):
         response = send_ollama_query("Hello, Ollama!")
         self.assertIsInstance(response, str)
 
+    def test_extract_search_terms(self):
+        from llm import extract_search_terms
+        # Test with capitalized terms
+        terms = extract_search_terms("What is Python programming?")
+        self.assertIsInstance(terms, list)
+        self.assertTrue(len(terms) > 0)
+        # Test with multiple capitalized words
+        terms2 = extract_search_terms("Tell me about Albert Einstein and Marie Curie")
+        self.assertIsInstance(terms2, list)
+        self.assertTrue(len(terms2) > 0)
+
+    def test_get_wiki_context(self):
+        from llm import get_wiki_context
+        # Test with a well-known topic
+        context = get_wiki_context("Python programming language")
+        self.assertIsInstance(context, str)
+        # Context might be empty if wiki is disabled or fails, that's ok
+
     def test_get_moon_phase(self):
         from space import get_moon
         phase = get_moon(lat, lon)

modules/wiki.py

@@ -23,7 +23,7 @@ def text_from_html(body):
     visible_texts = filter(tag_visible, texts)
     return " ".join(t.strip() for t in visible_texts if t.strip())
 
-def get_kiwix_summary(search_term):
+def get_kiwix_summary(search_term, truncate=True):
     """Query local Kiwix server for Wikipedia article"""
     if search_term is None or search_term.strip() == "":
         return ERROR_FETCHING_DATA
@@ -45,16 +45,23 @@ def get_kiwix_summary(search_term):
             summary = '. '.join(sentences[:wiki_return_limit])
             if summary and not summary.endswith('.'):
                 summary += '.'
-            return summary.strip()[:500]  # Hard limit at 500 chars
+            if truncate:
+                return summary.strip()[:500]  # Hard limit at 500 chars
+            else:
+                return summary.strip()
+        else:
+            logger.debug(f"System: Kiwix Library:{kiwix_library_name} failed for:{search_term} with status code {response.status_code}")
 
         # If direct access fails, try search
-        logger.debug(f"System: Kiwix direct article not found for:{search_term} Status Code:{response.status_code}")
         search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}"
         response = requests.get(search_url, timeout=urlTimeoutSeconds)
         
         if response.status_code == 200 and "No results were found" not in response.text:
             soup = bs.BeautifulSoup(response.text, 'html.parser')
             links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']]
+        else:
+            links = []
+            logger.debug(f"System: Kiwix Search failed for:{search_term} with status code {response.status_code}")
             
             for link in links[:3]:  # Check first 3 results
                 article_name = link.split("/")[-1]
@@ -71,7 +78,10 @@ def get_kiwix_summary(search_term):
                     summary = '. '.join(sentences[:wiki_return_limit])
                     if summary and not summary.endswith('.'):
                         summary += '.'
-                    return summary.strip()[:500]
+                    if truncate:
+                        return summary.strip()[:500]
+                    else:
+                        return summary.strip()
         
         logger.warning(f"System: No Kiwix Results for:{search_term}")
         # try to fall back to online Wikipedia if available
@@ -87,7 +97,7 @@ def get_kiwix_summary(search_term):
         logger.warning(f"System: Error with Kiwix for:{search_term} {e}")
         return ERROR_FETCHING_DATA
 
-def get_wikipedia_summary(search_term, location=None, force=False):
+def get_wikipedia_summary(search_term, location=None, force=False, truncate=True):
     if use_kiwix_server and not force:
         return get_kiwix_summary(search_term)
 
@@ -120,7 +130,11 @@ def get_wikipedia_summary(search_term, location=None, force=False):
         summary = '. '.join(sentences[:wiki_return_limit])
         if summary and not summary.endswith('.'):
             summary += '.'
-        return summary.strip()[:500]
+        if truncate:
+            # Truncate to 500 characters
+            return summary.strip()[:500]
+        else:
+            return summary.strip()
     except Exception as e:
         logger.warning(f"System: Wikipedia API error for:{search_term} {e}")
         return ERROR_FETCHING_DATA

requirements.txt

@@ -7,5 +7,4 @@ maidenhead
 beautifulsoup4
 dadjokes
 geopy
-schedule
-googlesearch-python
+schedule