diff --git a/README.md b/README.md index 40ecc0f..c96e847 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ git clone https://github.com/spudgunman/meshing-around | `messages` | Replays the last messages heard on device, like Store and Forward, returns the PublicChannel and Current | ✅ | | `readnews` | returns the contents of a file (data/news.txt, by default) can also `news mesh` via the chunker on air | ✅ | | `satpass` | returns the pass info from API for defined NORAD ID in config or Example: `satpass 25544,33591`| | -| `wiki:` | Searches Wikipedia and returns the first few sentences of the first result if a match. Example: `wiki: lora radio` | +| `wiki:` | Searches Wikipedia (or local Kiwix server) and returns the first few sentences of the first result if a match. Example: `wiki: lora radio` | | `howfar` | returns the distance you have traveled since your last HowFar. `howfar reset` to start over | ✅ | | `howtall` | returns height of something you give a shadow by using sun angle | ✅ | @@ -398,6 +398,33 @@ googleSearchResults = 3 # number of google search results to include in the cont ``` Note for LLM in docker with [NVIDIA](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html). Needed for the container with ollama running. +### Wikipedia Search Settings +The Wikipedia search module can use either the online Wikipedia API or a local Kiwix server for offline wiki access. Kiwix is especially useful for mesh networks operating in remote or offline environments. + +```ini +# Enable or disable the wikipedia search module +wikipedia = True + +# Use local Kiwix server instead of online Wikipedia +# Set to False to use online Wikipedia (default) +useKiwixServer = False + +# Kiwix server URL (only used if useKiwixServer is True) +kiwixURL = http://127.0.0.1:8080 + +# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06) +# Find available libraries at https://library.kiwix.org/ +kiwixLibraryName = wikipedia_en_100_nopic_2024-06 +``` + +To set up a local Kiwix server: +1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y` +2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim` +3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim` +4. Set `useKiwixServer = True` in your config.ini + +The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters). + ### Radio Monitoring A module allowing a Hamlib compatible radio to connect to the bot. When functioning, it will message the configured channel with a message of in use. **Requires hamlib/rigctld to be running as a service.** diff --git a/config.template b/config.template index 5bf1201..6a804c1 100644 --- a/config.template +++ b/config.template @@ -57,6 +57,13 @@ spaceWeather = True # enable or disable the wikipedia search module wikipedia = True +# Use local Kiwix server instead of online Wikipedia +# Set to False to use online Wikipedia, or provide Kiwix server URL +useKiwixServer = False +# Kiwix server URL (e.g., http://127.0.0.1:8080) +kiwixURL = http://127.0.0.1:8080 +# Kiwix library name (e.g., wikipedia_en_100_nopic_2025-09) +kiwixLibraryName = wikipedia_en_100_nopic_2025-09 # Enable ollama LLM see more at https://ollama.com ollama = False diff --git a/modules/settings.py b/modules/settings.py index fa49756..6360eff 100644 --- a/modules/settings.py +++ b/modules/settings.py @@ -227,6 +227,9 @@ try: bee_enabled = config['general'].getboolean('bee', False) # 🐝 off by default undocumented solar_conditions_enabled = config['general'].getboolean('spaceWeather', True) wikipedia_enabled = config['general'].getboolean('wikipedia', False) + use_kiwix_server = config['general'].getboolean('useKiwixServer', False) + kiwix_url = config['general'].get('kiwixURL', 'http://127.0.0.1:8080') + kiwix_library_name = config['general'].get('kiwixLibraryName', 'wikipedia_en_100_nopic_2024-06') llm_enabled = config['general'].getboolean('ollama', False) # https://ollama.com ollamaHostName = config['general'].get('ollamaHostName', 'http://localhost:11434') # default localhost llmModel = config['general'].get('ollamaModel', 'gemma3:270m') # default gemma3:270m diff --git a/modules/system.py b/modules/system.py index 90a0a28..7a9803a 100644 --- a/modules/system.py +++ b/modules/system.py @@ -206,8 +206,8 @@ if dad_jokes_enabled: # Wikipedia Search Configuration if wikipedia_enabled: - import wikipedia # pip install wikipedia - trap_list = trap_list + ("wiki:", "wiki?",) + from modules.wiki import * # from the spudgunman/meshing-around repo + trap_list = trap_list + ("wiki:",) help_message = help_message + ", wiki:" # LLM Configuration @@ -753,31 +753,6 @@ def send_message(message, ch, nodeid=0, nodeInt=1, bypassChuncking=False): interface.sendText(text=message, channelIndex=ch, destinationId=nodeid) return True -def get_wikipedia_summary(search_term): - wikipedia_search = wikipedia.search(search_term, results=3) - wikipedia_suggest = wikipedia.suggest(search_term) - #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) - #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") - - if len(wikipedia_search) == 0: - logger.warning(f"System: No Wikipedia Results for:{search_term}") - return ERROR_FETCHING_DATA - - try: - logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}") - summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True) - except wikipedia.DisambiguationError as e: - logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}") - summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) - except wikipedia.PageError as e: - logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}") - summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) - except Exception as e: - logger.warning(f"System: Error with Wikipedia for:{search_term} {e}") - return ERROR_FETCHING_DATA - - return summary - def messageTrap(msg): # Check if the message contains a trap word, this is the first filter for listning to messages # after this the message is passed to the command_handler in the bot.py which is switch case filter for applying word to function diff --git a/modules/wiki.py b/modules/wiki.py new file mode 100644 index 0000000..4fdac0d --- /dev/null +++ b/modules/wiki.py @@ -0,0 +1,122 @@ +# meshbot wiki module + +from modules.log import * +import wikipedia # pip install wikipedia + +# Kiwix support for local wiki +if use_kiwix_server: + import requests + from bs4 import BeautifulSoup + from urllib.parse import quote + from bs4.element import Comment + +# Kiwix helper functions (only loaded if use_kiwix_server is True) +if wikipedia_enabled and use_kiwix_server: + def tag_visible(element): + """Filter visible text from HTML elements for Kiwix""" + if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']: + return False + if isinstance(element, Comment): + return False + return True + + def text_from_html(body): + """Extract visible text from HTML content""" + soup = BeautifulSoup(body, 'html.parser') + texts = soup.find_all(string=True) + visible_texts = filter(tag_visible, texts) + return " ".join(t.strip() for t in visible_texts if t.strip()) + + def get_kiwix_summary(search_term): + """Query local Kiwix server for Wikipedia article""" + try: + search_encoded = quote(search_term) + # Try direct article access first + wiki_article = search_encoded.capitalize().replace("%20", "_") + exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}" + + response = requests.get(exact_url, timeout=urlTimeoutSeconds) + if response.status_code == 200: + # Extract and clean text + text = text_from_html(response.text) + # Remove common Wikipedia metadata prefixes + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + # Truncate to reasonable length (first few sentences) + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] # Hard limit at 500 chars + + # If direct access fails, try search + search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" + response = requests.get(search_url, timeout=urlTimeoutSeconds) + + if response.status_code == 200 and "No results were found" not in response.text: + soup = BeautifulSoup(response.text, 'html.parser') + links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] + + for link in links[:3]: # Check first 3 results + article_name = link.split("/")[-1] + if not article_name or article_name[0].islower(): + continue + + article_url = f"{kiwix_url}{link}" + article_response = requests.get(article_url, timeout=urlTimeoutSeconds) + if article_response.status_code == 200: + text = text_from_html(article_response.text) + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] + + logger.warning(f"System: No Kiwix Results for:{search_term}") + # try to fall back to online Wikipedia if available + return get_wikipedia_summary(search_term, force=True) + + + except requests.RequestException as e: + logger.warning(f"System: Kiwix connection error: {e}") + return "Unable to connect to local wiki server" + except Exception as e: + logger.warning(f"System: Error with Kiwix for:{search_term} {e}") + return ERROR_FETCHING_DATA + +def get_wikipedia_summary(search_term, location=None, force=False): + lat, lon = location if location else (None, None) + # Use Kiwix if configured + if use_kiwix_server and not force: + return get_kiwix_summary(search_term) + + try: + # Otherwise use online Wikipedia + wikipedia_search = wikipedia.search(search_term, results=3) + wikipedia_suggest = wikipedia.suggest(search_term) + #wikipedia_aroundme = wikipedia.geosearch(lat,lon, results=3) + #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") + except Exception as e: + logger.debug(f"System: Wikipedia search error for:{search_term} {e}") + return ERROR_FETCHING_DATA + + if len(wikipedia_search) == 0: + logger.warning(f"System: No Wikipedia Results for:{search_term}") + return ERROR_FETCHING_DATA + + try: + logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}") + summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True) + except wikipedia.DisambiguationError as e: + logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}") + summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) + except wikipedia.PageError as e: + logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}") + summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) + except Exception as e: + logger.warning(f"System: Error with Wikipedia for:{search_term} {e}") + return ERROR_FETCHING_DATA + + return summary \ No newline at end of file