diff --git a/README.md b/README.md index 64b5cff..3b3769a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Mesh Bot is a feature-rich Python bot designed to enhance your [Meshtastic](http ### Interactive AI and Data Lookup - **Weather, Earthquake, River, and Tide Data**: Get local alerts and info from NOAA/USGS; uses Open-Meteo for areas outside NOAA coverage. - **Wikipedia Search**: Retrieve summaries from Wikipedia. -- **Ollama LLM Integration**: Query the [Ollama](https://github.com/ollama/ollama/tree/main/docs) AI for advanced responses. +- **Ollama LLM Integration**: Query the [Ollama](https://github.com/ollama/ollama/tree/main/docs) AI for advanced responses. Supports RAG (Retrieval Augmented Generation) with Wikipedia/Kiwix context and [OpenWebUI](https://github.com/open-webui/open-webui) integration for enhanced AI capabilities. - **Satellite Passes**: Find upcoming satellite passes for your location. - **GeoMeasuring Tools**: Calculate distances and midpoints using collected GPS data; supports Fox & Hound direction finding. diff --git a/config.template b/config.template index 36de8a4..c705252 100644 --- a/config.template +++ b/config.template @@ -75,15 +75,28 @@ kiwixLibraryName = wikipedia_en_100_nopic_2025-09 # Enable ollama LLM see more at https://ollama.com ollama = False -# Ollama model to use (defaults to gemma3:270m) +# Ollama model to use (defaults to gemma3:270m) gemma2 is good for older SYSTEM prompt # ollamaModel = gemma3:latest +# ollamaModel = gemma2:2b # server instance to use (defaults to local machine install) ollamaHostName = http://localhost:11434 + # Produce LLM replies to messages that aren't commands? # If False, the LLM only replies to the "ask:" and "askai" commands. llmReplyToNonCommands = True -# if True, the input is sent raw to the LLM, if False uses legacy template query -rawLLMQuery = True +# if True, the input is sent raw to the LLM, if False uses SYSTEM prompt +rawLLMQuery = True + +# Enable Wikipedia/Kiwix integration with LLM for RAG (Retrieval Augmented Generation) +# When enabled, LLM will automatically search Wikipedia/Kiwix and include context in responses +llmUseWikiContext = False + +# Use OpenWebUI instead of direct Ollama API (enables advanced RAG features) +useOpenWebUI = False +# OpenWebUI server URL (e.g., http://localhost:3000) +openWebUIURL = http://localhost:3000 +# OpenWebUI API key/token (required when useOpenWebUI is True) +openWebUIAPIKey = # StoreForward Enabled and Limits StoreForward = True diff --git a/mesh_bot.py b/mesh_bot.py index 1bb6fba..85becdf 100755 --- a/mesh_bot.py +++ b/mesh_bot.py @@ -1491,10 +1491,21 @@ def handle_boot(mesh=True): f"{get_name_from_number(myNodeNum, 'short', i)}. NodeID: {myNodeNum}, {decimal_to_hex(myNodeNum)}") if llm_enabled: - logger.debug(f"System: Ollama LLM Enabled, loading model {my_settings.llmModel} please wait") - llmLoad = llm_query(" ") + msg = f"System: LLM Enabled" + llmLoad = llm_query(" ", init=True) if "trouble" not in llmLoad: - logger.debug(f"System: LLM Model {my_settings.llmModel} loaded") + if my_settings.llmReplyToNonCommands: + msg += " | Reply to DM's Enabled" + if my_settings.llmUseWikiContext: + wiki_source = "Kiwixpedia" if my_settings.use_kiwix_server else "Wikipedia" + msg += f" | {wiki_source} Context Enabled" + if my_settings.useOpenWebUI: + msg += " | OpenWebUI API Enabled" + else: + msg += f" | Ollama API Model {my_settings.llmModel} loaded. Use {'RAW' if my_settings.rawLLMQuery else 'SYSTEM'} prompt mode." + logger.debug(msg) + else: + logger.debug(f"System: Bad response from LLM: {llmLoad}") if my_settings.bbs_enabled: logger.debug(f"System: BBS Enabled, {bbsdb} has {len(bbs_messages)} messages. Direct Mail Messages waiting: {(len(bbs_dm) - 1)}") diff --git a/modules/README.md b/modules/README.md index b103871..43b0406 100644 --- a/modules/README.md +++ b/modules/README.md @@ -808,7 +808,7 @@ To set up a local Kiwix server: 1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y` 2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim` 3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim` -4. Set `useKiwixServer = True` in your config.ini +4. Set `useKiwixServer = True` in your config.ini with `wikipedia = True` The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters). diff --git a/modules/llm.md b/modules/llm.md new file mode 100644 index 0000000..cfdf2c6 --- /dev/null +++ b/modules/llm.md @@ -0,0 +1,52 @@ +# How do I use this thing? +This is not a full turnkey setup for Docker yet? + + +# Ollama local +```bash +# bash +curl -fsSL https://ollama.com/install.sh | sh +# docker +docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -e OLLAMA_API_BASE_URL=http://host.docker.internal:11434 open-webui/open-webui +``` + +```ini +#service file addition +# https://github.com/ollama/ollama/issues/703 +[Service] +Environment="OLLAMA_HOST=0.0.0.0:11434" +``` +## validation +http://IP::11434 +`Ollama is running` + +# OpenWebUI (docker) +```bash +## ollama in docker +docker run -d -p 3000:8080 --gpus all -v open-webui:/app/backend/data --name open-webui ghcr.io/open-webui/open-webui:cuda + +## external ollama +docker run -d -p 3000:8080 -e OLLAMA_BASE_URL=https://IP:11434 -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main +``` +wait for engine to build, update the config.ini for the bot + +```ini +# Use OpenWebUI instead of direct Ollama API (enables advanced RAG features) +useOpenWebUI = True +# OpenWebUI server URL (e.g., http://localhost:3000) +openWebUIURL = http://IP:3000 +``` + +## validation +http://IP:3000 +make a new admin user. +validate you have models imported or that the system is working for query. +make a new user for the bot + + +upper right settings for the user +settings -> account +get/create the API key for the user + + +set api endpoint [OpenWebUI API](https://docs.openwebui.com/getting-started/api-endpoints) \ No newline at end of file diff --git a/modules/llm.py b/modules/llm.py index 81d6aad..eb90bae 100644 --- a/modules/llm.py +++ b/modules/llm.py @@ -3,7 +3,8 @@ # This module is used to interact with LLM API to generate responses to user input # K7MHI Kelly Keeton 2024 from modules.log import logger -from modules.settings import llmModel, ollamaHostName, rawLLMQuery +from modules.settings import (llmModel, ollamaHostName, rawLLMQuery, + llmUseWikiContext, useOpenWebUI, openWebUIURL, openWebUIAPIKey, cmdBang, urlTimeoutSeconds, use_kiwix_server) # Ollama Client # https://github.com/ollama/ollama/blob/main/docs/faq.md#how-do-i-configure-ollama-server @@ -11,22 +12,17 @@ import requests import json from datetime import datetime -if not rawLLMQuery: - # this may be removed in the future - from googlesearch import search # pip install googlesearch-python - # LLM System Variables ollamaAPI = ollamaHostName + "/api/generate" +openWebUIChatAPI = openWebUIURL + "/api/chat/completions" +openWebUIOllamaProxy = openWebUIURL + "/ollama/api/generate" tokens = 450 # max charcters for the LLM response, this is the max length of the response also in prompts -requestTruncation = True # if True, the LLM "will" truncate the response - -openaiAPI = "https://api.openai.com/v1/completions" # not used, if you do push a enhancement! +requestTruncation = True # if True, the LLM "will" truncate the response +DEBUG_LLM = False # enable debug logging for LLM queries # Used in the meshBotAI template llmEnableHistory = True # enable last message history for the LLM model -llmContext_fromGoogle = True # enable context from google search results adds to compute time but really helps with responses accuracy -googleSearchResults = 3 # number of google search results to include in the context more results = more compute time antiFloodLLM = [] llmChat_history = {} trap_list_llm = ("ask:", "askai") @@ -52,24 +48,6 @@ meshBotAI = """ """ -if llmContext_fromGoogle: - meshBotAI = meshBotAI + """ - CONTEXT - The following is the location of the user - {location_name} - - The following is for context around the prompt to help guide your response. - {context} - - """ -else: - meshBotAI = meshBotAI + """ - CONTEXT - The following is the location of the user - {location_name} - - """ - if llmEnableHistory: meshBotAI = meshBotAI + """ HISTORY @@ -101,22 +79,6 @@ def llmTool_math_calculator(expression): except Exception as e: return f"Error in calculation: {e}" -def llmTool_get_google(query, num_results=3): - """ - Example tool function to perform a Google search and return results. - :param query: The search query string. - :param num_results: Number of search results to return. - :return: A list of search result titles and descriptions. - """ - results = [] - try: - googleSearch = search(query, advanced=True, num_results=num_results) - for result in googleSearch: - results.append(f"{result.title}: {result.description}") - return results - except Exception as e: - return [f"Error in Google search: {e}"] - llmFunctions = [ { @@ -141,46 +103,164 @@ llmFunctions = [ "required": ["expression"] } }, - { - "name": "llmTool_get_google", - "description": "Perform a Google search and return results.", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "The search query string." - }, - "num_results": { - "type": "integer", - "description": "Number of search results to return.", - "default": 3 - } - }, - "required": ["query"] - } - } ] -def get_google_context(input, num_results): - # Get context from Google search results - googleResults = [] +def get_wiki_context(input): + """ + Get context from Wikipedia/Kiwix for RAG enhancement + :param input: The user query + :return: Wikipedia summary or empty string if not available + """ try: - googleSearch = search(input, advanced=True, num_results=num_results) - if googleSearch: - for result in googleSearch: - googleResults.append(f"{result.title} {result.description}") - else: - googleResults = ['no other context provided'] + from modules.wiki import get_wikipedia_summary, get_kiwix_summary + # Extract potential search terms from the input + # Try to identify key topics/entities for Wikipedia search + search_terms = extract_search_terms(input) + + wiki_context = [] + for term in search_terms[:2]: # Limit to 2 searches to avoid excessive API calls + if use_kiwix_server: + summary = get_kiwix_summary(term, truncate=False) + else: + summary = get_wikipedia_summary(term, truncate=False) + + if summary and "error" not in summary.lower(): + wiki_context.append(f"Wikipedia context for '{term}': {summary}") + + return '\n'.join(wiki_context) if wiki_context else '' except Exception as e: - logger.debug(f"System: LLM Query: context gathering failed, likely due to network issues") - googleResults = ['no other context provided'] - return googleResults + logger.debug(f"System: LLM Query: Wiki context gathering failed: {e}") + return '' + +def llm_extract_topic(input): + """ + Use LLM to extract the main topic as a single word or short phrase. + Always uses raw mode and supports both Ollama and OpenWebUI. + :param input: The user query + :return: List with one topic string, or empty list on failure + """ + prompt = ( + "Summarize the following query into a single word or short phrase that best represents the main topic, " + "for use as a Wikipedia search term. Only return the word or phrase, nothing else:\n" + f"{input}" + ) + try: + if useOpenWebUI and openWebUIAPIKey: + result = send_openwebui_query(prompt, max_tokens=10) + else: + llmQuery = {"model": llmModel, "prompt": prompt, "stream": False, "max_tokens": 10} + result = send_ollama_query(llmQuery) + topic = result.strip().split('\n')[0] + topic = topic.strip(' "\'.,!?;:') + if topic: + return [topic] + except Exception as e: + logger.debug(f"LLM topic extraction failed: {e}") + return [] + +def extract_search_terms(input): + """ + Extract potential search terms from user input. + Enhanced: Try LLM-based topic extraction first, fallback to heuristic. + :param input: The user query + :return: List of potential search terms + """ + # Remove common command prefixes + for trap in trap_list_llm: + if input.lower().startswith(trap): + input = input[len(trap):].strip() + break + + # Try LLM-based extraction first + terms = llm_extract_topic(input) + if terms: + return terms + + # Fallback: Simple heuristic (existing code) + words = input.split() + search_terms = [] + temp_phrase = [] + for word in words: + clean_word = word.strip('.,!?;:') + if clean_word and clean_word[0].isupper() and len(clean_word) > 2: + temp_phrase.append(clean_word) + elif temp_phrase: + search_terms.append(' '.join(temp_phrase)) + temp_phrase = [] + if temp_phrase: + search_terms.append(' '.join(temp_phrase)) + if not search_terms: + search_terms = [input.strip()] + if DEBUG_LLM: + logger.debug(f"Extracted search terms: {search_terms}") + return search_terms[:3] # Limit to 3 terms + +def send_openwebui_query(prompt, model=None, max_tokens=450, context=''): + """ + Send query to OpenWebUI API for chat completion + :param prompt: The user prompt + :param model: Model name (optional, defaults to llmModel) + :param max_tokens: Max tokens for response + :param context: Additional context to include + :return: Response text or error message + """ + if model is None: + model = llmModel + + headers = { + 'Authorization': f'Bearer {openWebUIAPIKey}', + 'Content-Type': 'application/json' + } + + messages = [] + if context: + messages.append({ + "role": "system", + "content": f"Use the following context to help answer questions:\n{context}" + }) + + messages.append({ + "role": "user", + "content": prompt + }) + + data = { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "stream": False + } + + # Debug logging + if DEBUG_LLM: + logger.debug(f"OpenWebUI payload: {json.dumps(data)}") + logger.debug(f"OpenWebUI endpoint: {openWebUIChatAPI}") + + try: + result = requests.post(openWebUIChatAPI, headers=headers, json=data, timeout=urlTimeoutSeconds * 5) + if DEBUG_LLM: + logger.debug(f"OpenWebUI response status: {result.status_code}") + logger.debug(f"OpenWebUI response text: {result.text}") + if result.status_code == 200: + result_json = result.json() + # OpenWebUI returns OpenAI-compatible format + if 'choices' in result_json and len(result_json['choices']) > 0: + response = result_json['choices'][0]['message']['content'] + return response.strip() + else: + logger.warning(f"System: OpenWebUI API returned unexpected format") + return "⛔️ Response Error" + else: + logger.warning(f"System: OpenWebUI API returned status code {result.status_code}") + return f"⛔️ Request Error" + except requests.exceptions.RequestException as e: + logger.warning(f"System: OpenWebUI API request failed: {e}") + return f"⛔️ Request Error" def send_ollama_query(llmQuery): # Send the query to the Ollama API and return the response try: - result = requests.post(ollamaAPI, data=json.dumps(llmQuery), timeout=5) + result = requests.post(ollamaAPI, data=json.dumps(llmQuery), timeout= urlTimeoutSeconds * 5) if result.status_code == 200: result_json = result.json() result = result_json.get("response", "") @@ -219,24 +299,28 @@ def send_ollama_tooling_query(prompt, functions, model=None, max_tokens=450): else: raise Exception(f"HTTP Error: {result.status_code} - {result.text}") -def llm_query(input, nodeID=0, location_name=None): +def llm_query(input, nodeID=0, location_name=None, init=False): global antiFloodLLM, llmChat_history - googleResults = [] + wikiContext = '' # if this is the first initialization of the LLM the query of " " should bring meshbotAIinit OTA shouldnt reach this? # This is for LLM like gemma and others now? - if input == " " and rawLLMQuery: + if init and rawLLMQuery: logger.warning("System: These LLM models lack a traditional system prompt, they can be verbose and not very helpful be advised.") input = meshbotAIinit - else: + elif init: input = input.strip() # classic model for gemma2, deepseek-r1, etc - logger.debug(f"System: Using classic LLM model framework, ideally for gemma2, deepseek-r1, etc") + logger.debug(f"System: Using SYSTEM model framework, ideally for gemma2, deepseek-r1, etc") if not location_name: location_name = "no location provided " + + # Remove command bang if present + if cmdBang and input.startswith('!'): + input = input.strip('!').strip() - # remove askai: and ask: from the input + # Remove any trap words from the start of the input for trap in trap_list_llm: if input.lower().startswith(trap): input = input[len(trap):].strip() @@ -251,34 +335,84 @@ def llm_query(input, nodeID=0, location_name=None): else: antiFloodLLM.append(nodeID) - if llmContext_fromGoogle and not rawLLMQuery: - googleResults = get_google_context(input, googleSearchResults) + # Get Wikipedia/Kiwix context if enabled (RAG) + if llmUseWikiContext and input != meshbotAIinit: + # get_wiki_context returns a string, but we want to count the items before joining + search_terms = extract_search_terms(input) + wiki_context_list = [] + for term in search_terms[:2]: + if not use_kiwix_server: + summary = get_wiki_context(term) + else: + summary = get_wiki_context(term) + if summary and "error" not in summary.lower(): + wiki_context_list.append(f"Wikipedia context for '{term}': {summary}") + wikiContext = '\n'.join(wiki_context_list) if wiki_context_list else '' + if wikiContext: + logger.debug(f"System: using Wikipedia/Kiwix context for LLM query got {len(wiki_context_list)} results") history = llmChat_history.get(nodeID, ["", ""]) - - if googleResults: - logger.debug(f"System: Google-Enhanced LLM Query: {input} From:{nodeID}") - else: - logger.debug(f"System: LLM Query: {input} From:{nodeID}") response = "" result = "" location_name += f" at the current time of {datetime.now().strftime('%Y-%m-%d %H:%M:%S %Z')}" try: - if rawLLMQuery: - # sanitize the input to remove tool call syntax - if '```' in input: - logger.warning("System: LLM Query: Code markdown detected, removing for raw query") - input = input.replace('```bash', '').replace('```python', '').replace('```', '') - modelPrompt = input - else: - # Build the query from the template - modelPrompt = meshBotAI.format(input=input, context='\n'.join(googleResults), location_name=location_name, llmModel=llmModel, history=history) + # Use OpenWebUI if enabled + if useOpenWebUI and openWebUIAPIKey: + logger.debug(f"System: LLM Query: Using OpenWebUI API for LLM query {input} From:{nodeID}") - llmQuery = {"model": llmModel, "prompt": modelPrompt, "stream": False, "max_tokens": tokens} - # Query the model via Ollama web API - result = send_ollama_query(llmQuery) + # Combine all context sources + combined_context = [] + if wikiContext: + combined_context.append(wikiContext) + + context_str = '\n\n'.join(combined_context) + + # For OpenWebUI, we send a cleaner prompt + if rawLLMQuery: + result = send_openwebui_query(input, context=context_str, max_tokens=tokens) + else: + # Use the template for non-raw queries + modelPrompt = meshBotAI.format( + input=input, + context=context_str if combined_context else 'no other context provided', + location_name=location_name, + llmModel=llmModel, + history=history + ) + result = send_openwebui_query(modelPrompt, max_tokens=tokens) + else: + logger.debug(f"System: LLM Query: Using Ollama API for LLM query {input} From:{nodeID}") + # Use standard Ollama API + if rawLLMQuery: + # sanitize the input to remove tool call syntax + if '```' in input: + logger.warning("System: LLM Query: Code markdown detected, removing for raw query") + input = input.replace('```bash', '').replace('```python', '').replace('```', '') + modelPrompt = input + + # Add wiki context to raw queries if available + if wikiContext: + modelPrompt = f"Context:\n{wikiContext}\n\nQuestion: {input}" + else: + # Build the query from the template + all_context = [] + if wikiContext: + all_context.append(wikiContext) + + context_text = '\n'.join(all_context) if all_context else 'no other context provided' + modelPrompt = meshBotAI.format( + input=input, + context=context_text, + location_name=location_name, + llmModel=llmModel, + history=history + ) + + llmQuery = {"model": llmModel, "prompt": modelPrompt, "stream": False, "max_tokens": tokens} + # Query the model via Ollama web API + result = send_ollama_query(llmQuery) #logger.debug(f"System: LLM Response: " + result.strip().replace('\n', ' ')) except Exception as e: @@ -290,13 +424,17 @@ def llm_query(input, nodeID=0, location_name=None): response = result.strip().replace('\n', ' ') if rawLLMQuery and requestTruncation and len(response) > 450: - #retryy loop to truncate the response + # retry loop to truncate the response logger.warning(f"System: LLM Query: Response exceeded {tokens} characters, requesting truncation") - truncateQuery = {"model": llmModel, "prompt": truncatePrompt + response, "stream": False, "max_tokens": tokens} - truncateResult = send_ollama_query(truncateQuery) + truncate_prompt_full = truncatePrompt + response + if useOpenWebUI and openWebUIAPIKey: + truncateResult = send_openwebui_query(truncate_prompt_full, max_tokens=tokens) + else: + truncateQuery = {"model": llmModel, "prompt": truncate_prompt_full, "stream": False, "max_tokens": tokens} + truncateResult = send_ollama_query(truncateQuery) # cleanup for message output - response = result.strip().replace('\n', ' ') + response = truncateResult.strip().replace('\n', ' ') # done with the query, remove the user from the anti flood list antiFloodLLM.remove(nodeID) diff --git a/modules/settings.py b/modules/settings.py index 4f7a6b6..ae89ae1 100644 --- a/modules/settings.py +++ b/modules/settings.py @@ -256,6 +256,10 @@ try: llmModel = config['general'].get('ollamaModel', 'gemma3:270m') # default gemma3:270m rawLLMQuery = config['general'].getboolean('rawLLMQuery', True) #default True llmReplyToNonCommands = config['general'].getboolean('llmReplyToNonCommands', True) # default True + llmUseWikiContext = config['general'].getboolean('llmUseWikiContext', False) # default False + useOpenWebUI = config['general'].getboolean('useOpenWebUI', False) # default False + openWebUIURL = config['general'].get('openWebUIURL', 'http://localhost:3000') # default localhost:3000 + openWebUIAPIKey = config['general'].get('openWebUIAPIKey', '') # default empty dont_retry_disconnect = config['general'].getboolean('dont_retry_disconnect', False) # default False, retry on disconnect favoriteNodeList = config['general'].get('favoriteNodeList', '').split(',') enableEcho = config['general'].getboolean('enableEcho', False) # default False diff --git a/modules/test_bot.py b/modules/test_bot.py index ae101dd..543edcd 100644 --- a/modules/test_bot.py +++ b/modules/test_bot.py @@ -97,6 +97,24 @@ class TestBot(unittest.TestCase): response = send_ollama_query("Hello, Ollama!") self.assertIsInstance(response, str) + def test_extract_search_terms(self): + from llm import extract_search_terms + # Test with capitalized terms + terms = extract_search_terms("What is Python programming?") + self.assertIsInstance(terms, list) + self.assertTrue(len(terms) > 0) + # Test with multiple capitalized words + terms2 = extract_search_terms("Tell me about Albert Einstein and Marie Curie") + self.assertIsInstance(terms2, list) + self.assertTrue(len(terms2) > 0) + + def test_get_wiki_context(self): + from llm import get_wiki_context + # Test with a well-known topic + context = get_wiki_context("Python programming language") + self.assertIsInstance(context, str) + # Context might be empty if wiki is disabled or fails, that's ok + def test_get_moon_phase(self): from space import get_moon phase = get_moon(lat, lon) diff --git a/modules/wiki.py b/modules/wiki.py index 37e2522..b40b1f6 100644 --- a/modules/wiki.py +++ b/modules/wiki.py @@ -23,7 +23,7 @@ def text_from_html(body): visible_texts = filter(tag_visible, texts) return " ".join(t.strip() for t in visible_texts if t.strip()) -def get_kiwix_summary(search_term): +def get_kiwix_summary(search_term, truncate=True): """Query local Kiwix server for Wikipedia article""" if search_term is None or search_term.strip() == "": return ERROR_FETCHING_DATA @@ -45,16 +45,23 @@ def get_kiwix_summary(search_term): summary = '. '.join(sentences[:wiki_return_limit]) if summary and not summary.endswith('.'): summary += '.' - return summary.strip()[:500] # Hard limit at 500 chars + if truncate: + return summary.strip()[:500] # Hard limit at 500 chars + else: + return summary.strip() + else: + logger.debug(f"System: Kiwix Library:{kiwix_library_name} failed for:{search_term} with status code {response.status_code}") # If direct access fails, try search - logger.debug(f"System: Kiwix direct article not found for:{search_term} Status Code:{response.status_code}") search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" response = requests.get(search_url, timeout=urlTimeoutSeconds) if response.status_code == 200 and "No results were found" not in response.text: soup = bs.BeautifulSoup(response.text, 'html.parser') links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] + else: + links = [] + logger.debug(f"System: Kiwix Search failed for:{search_term} with status code {response.status_code}") for link in links[:3]: # Check first 3 results article_name = link.split("/")[-1] @@ -71,7 +78,10 @@ def get_kiwix_summary(search_term): summary = '. '.join(sentences[:wiki_return_limit]) if summary and not summary.endswith('.'): summary += '.' - return summary.strip()[:500] + if truncate: + return summary.strip()[:500] + else: + return summary.strip() logger.warning(f"System: No Kiwix Results for:{search_term}") # try to fall back to online Wikipedia if available @@ -87,7 +97,7 @@ def get_kiwix_summary(search_term): logger.warning(f"System: Error with Kiwix for:{search_term} {e}") return ERROR_FETCHING_DATA -def get_wikipedia_summary(search_term, location=None, force=False): +def get_wikipedia_summary(search_term, location=None, force=False, truncate=True): if use_kiwix_server and not force: return get_kiwix_summary(search_term) @@ -120,7 +130,11 @@ def get_wikipedia_summary(search_term, location=None, force=False): summary = '. '.join(sentences[:wiki_return_limit]) if summary and not summary.endswith('.'): summary += '.' - return summary.strip()[:500] + if truncate: + # Truncate to 500 characters + return summary.strip()[:500] + else: + return summary.strip() except Exception as e: logger.warning(f"System: Wikipedia API error for:{search_term} {e}") return ERROR_FETCHING_DATA diff --git a/requirements.txt b/requirements.txt index 4b8d80b..1b70d42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,4 @@ maidenhead beautifulsoup4 dadjokes geopy -schedule -googlesearch-python +schedule \ No newline at end of file