From 413f2a24d9970086d1be15150df81147bb9dd64f Mon Sep 17 00:00:00 2001 From: SpudGunMan Date: Wed, 8 Oct 2025 20:29:52 -0700 Subject: [PATCH] Kiwix in wiki @NomDeTom its finally done --- README.md | 6 +-- config.template | 4 +- modules/system.py | 115 +------------------------------------------- modules/wiki.py | 119 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 118 deletions(-) create mode 100644 modules/wiki.py diff --git a/README.md b/README.md index 15e14f1..772cf57 100644 --- a/README.md +++ b/README.md @@ -417,9 +417,9 @@ kiwixLibraryName = wikipedia_en_100_nopic_2024-06 ``` To set up a local Kiwix server: -1. Install Kiwix tools: https://kiwix.org/en/ -2. Download a Wikipedia ZIM file: https://library.kiwix.org/ -3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2024-06.zim` +1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y` +2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim` +3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim` 4. Set `useKiwixServer = True` in your config.ini The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters). diff --git a/config.template b/config.template index 0e8a13c..421481f 100644 --- a/config.template +++ b/config.template @@ -62,8 +62,8 @@ wikipedia = True useKiwixServer = False # Kiwix server URL (e.g., http://127.0.0.1:8080) kiwixURL = http://127.0.0.1:8080 -# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06) -kiwixLibraryName = wikipedia_en_100_nopic_2024-06 +# Kiwix library name (e.g., wikipedia_en_100_nopic_2025-09) +kiwixLibraryName = wikipedia_en_100_nopic_2025-09 # Enable ollama LLM see more at https://ollama.com ollama = False diff --git a/modules/system.py b/modules/system.py index 1a6dce0..67811de 100644 --- a/modules/system.py +++ b/modules/system.py @@ -206,16 +206,9 @@ if dad_jokes_enabled: # Wikipedia Search Configuration if wikipedia_enabled: - import wikipedia # pip install wikipedia - trap_list = trap_list + ("wiki:", "wiki?",) + from modules.wiki import * # from the spudgunman/meshing-around repo + trap_list = trap_list + ("wiki:",) help_message = help_message + ", wiki:" - - # Kiwix support for local wiki - if use_kiwix_server: - import requests - from bs4 import BeautifulSoup - from urllib.parse import quote - from bs4.element import Comment # LLM Configuration if llm_enabled: @@ -760,110 +753,6 @@ def send_message(message, ch, nodeid=0, nodeInt=1, bypassChuncking=False): interface.sendText(text=message, channelIndex=ch, destinationId=nodeid) return True -# Kiwix helper functions (only loaded if use_kiwix_server is True) -if wikipedia_enabled and use_kiwix_server: - def tag_visible(element): - """Filter visible text from HTML elements for Kiwix""" - if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']: - return False - if isinstance(element, Comment): - return False - return True - - def text_from_html(body): - """Extract visible text from HTML content""" - soup = BeautifulSoup(body, 'html.parser') - texts = soup.find_all(string=True) - visible_texts = filter(tag_visible, texts) - return " ".join(t.strip() for t in visible_texts if t.strip()) - - def get_kiwix_summary(search_term): - """Query local Kiwix server for Wikipedia article""" - try: - search_encoded = quote(search_term) - # Try direct article access first - wiki_article = search_encoded.capitalize().replace("%20", "_") - exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}" - - response = requests.get(exact_url, timeout=urlTimeoutSeconds) - if response.status_code == 200: - # Extract and clean text - text = text_from_html(response.text) - # Remove common Wikipedia metadata prefixes - text = text.split("Jump to navigation", 1)[-1] - text = text.split("Jump to search", 1)[-1] - # Truncate to reasonable length (first few sentences) - sentences = text.split('. ') - summary = '. '.join(sentences[:wiki_return_limit]) - if summary and not summary.endswith('.'): - summary += '.' - return summary.strip()[:500] # Hard limit at 500 chars - - # If direct access fails, try search - search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" - response = requests.get(search_url, timeout=urlTimeoutSeconds) - - if response.status_code == 200 and "No results were found" not in response.text: - soup = BeautifulSoup(response.text, 'html.parser') - links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] - - for link in links[:3]: # Check first 3 results - article_name = link.split("/")[-1] - if not article_name or article_name[0].islower(): - continue - - article_url = f"{kiwix_url}{link}" - article_response = requests.get(article_url, timeout=urlTimeoutSeconds) - if article_response.status_code == 200: - text = text_from_html(article_response.text) - text = text.split("Jump to navigation", 1)[-1] - text = text.split("Jump to search", 1)[-1] - sentences = text.split('. ') - summary = '. '.join(sentences[:wiki_return_limit]) - if summary and not summary.endswith('.'): - summary += '.' - return summary.strip()[:500] - - logger.warning(f"System: No Kiwix Results for:{search_term}") - return ERROR_FETCHING_DATA - - except requests.RequestException as e: - logger.warning(f"System: Kiwix connection error: {e}") - return "Unable to connect to local wiki server" - except Exception as e: - logger.warning(f"System: Error with Kiwix for:{search_term} {e}") - return ERROR_FETCHING_DATA - -def get_wikipedia_summary(search_term): - # Use Kiwix if configured - if use_kiwix_server: - return get_kiwix_summary(search_term) - - # Otherwise use online Wikipedia - wikipedia_search = wikipedia.search(search_term, results=3) - wikipedia_suggest = wikipedia.suggest(search_term) - #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) - #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") - - if len(wikipedia_search) == 0: - logger.warning(f"System: No Wikipedia Results for:{search_term}") - return ERROR_FETCHING_DATA - - try: - logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}") - summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True) - except wikipedia.DisambiguationError as e: - logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}") - summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) - except wikipedia.PageError as e: - logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}") - summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) - except Exception as e: - logger.warning(f"System: Error with Wikipedia for:{search_term} {e}") - return ERROR_FETCHING_DATA - - return summary - def messageTrap(msg): # Check if the message contains a trap word, this is the first filter for listning to messages # after this the message is passed to the command_handler in the bot.py which is switch case filter for applying word to function diff --git a/modules/wiki.py b/modules/wiki.py new file mode 100644 index 0000000..ecfd8cf --- /dev/null +++ b/modules/wiki.py @@ -0,0 +1,119 @@ +# meshbot wiki module + +from modules.log import * +import wikipedia # pip install wikipedia + +# Kiwix support for local wiki +if use_kiwix_server: + import requests + from bs4 import BeautifulSoup + from urllib.parse import quote + from bs4.element import Comment + +# Kiwix helper functions (only loaded if use_kiwix_server is True) +if wikipedia_enabled and use_kiwix_server: + def tag_visible(element): + """Filter visible text from HTML elements for Kiwix""" + if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']: + return False + if isinstance(element, Comment): + return False + return True + + def text_from_html(body): + """Extract visible text from HTML content""" + soup = BeautifulSoup(body, 'html.parser') + texts = soup.find_all(string=True) + visible_texts = filter(tag_visible, texts) + return " ".join(t.strip() for t in visible_texts if t.strip()) + + def get_kiwix_summary(search_term): + """Query local Kiwix server for Wikipedia article""" + try: + search_encoded = quote(search_term) + # Try direct article access first + wiki_article = search_encoded.capitalize().replace("%20", "_") + exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}" + + response = requests.get(exact_url, timeout=urlTimeoutSeconds) + if response.status_code == 200: + # Extract and clean text + text = text_from_html(response.text) + # Remove common Wikipedia metadata prefixes + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + # Truncate to reasonable length (first few sentences) + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] # Hard limit at 500 chars + + # If direct access fails, try search + search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" + response = requests.get(search_url, timeout=urlTimeoutSeconds) + + if response.status_code == 200 and "No results were found" not in response.text: + soup = BeautifulSoup(response.text, 'html.parser') + links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] + + for link in links[:3]: # Check first 3 results + article_name = link.split("/")[-1] + if not article_name or article_name[0].islower(): + continue + + article_url = f"{kiwix_url}{link}" + article_response = requests.get(article_url, timeout=urlTimeoutSeconds) + if article_response.status_code == 200: + text = text_from_html(article_response.text) + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] + + logger.warning(f"System: No Kiwix Results for:{search_term}") + return ERROR_FETCHING_DATA + + except requests.RequestException as e: + logger.warning(f"System: Kiwix connection error: {e}") + return "Unable to connect to local wiki server" + except Exception as e: + logger.warning(f"System: Error with Kiwix for:{search_term} {e}") + return ERROR_FETCHING_DATA + +def get_wikipedia_summary(search_term): + # Use Kiwix if configured + if use_kiwix_server: + return get_kiwix_summary(search_term) + + try: + # Otherwise use online Wikipedia + wikipedia_search = wikipedia.search(search_term, results=3) + wikipedia_suggest = wikipedia.suggest(search_term) + #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) + #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") + except Exception as e: + logger.debug(f"System: Wikipedia search error for:{search_term} {e}") + return ERROR_FETCHING_DATA + + if len(wikipedia_search) == 0: + logger.warning(f"System: No Wikipedia Results for:{search_term}") + return ERROR_FETCHING_DATA + + try: + logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}") + summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True) + except wikipedia.DisambiguationError as e: + logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}") + summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) + except wikipedia.PageError as e: + logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}") + summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) + except Exception as e: + logger.warning(f"System: Error with Wikipedia for:{search_term} {e}") + return ERROR_FETCHING_DATA + + return summary \ No newline at end of file