From c97004b41000476b1a4126739c164b6542e354f5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Oct 2025 15:00:44 +0000 Subject: [PATCH 2/5] Add Kiwix local wiki server support with configuration options Co-authored-by: SpudGunMan <12676665+SpudGunMan@users.noreply.github.com> --- config.template | 7 ++++ modules/settings.py | 3 ++ modules/system.py | 86 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/config.template b/config.template index e94c846..0e8a13c 100644 --- a/config.template +++ b/config.template @@ -57,6 +57,13 @@ spaceWeather = True # enable or disable the wikipedia search module wikipedia = True +# Use local Kiwix server instead of online Wikipedia +# Set to False to use online Wikipedia, or provide Kiwix server URL +useKiwixServer = False +# Kiwix server URL (e.g., http://127.0.0.1:8080) +kiwixURL = http://127.0.0.1:8080 +# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06) +kiwixLibraryName = wikipedia_en_100_nopic_2024-06 # Enable ollama LLM see more at https://ollama.com ollama = False diff --git a/modules/settings.py b/modules/settings.py index 82de6cf..d6c5346 100644 --- a/modules/settings.py +++ b/modules/settings.py @@ -226,6 +226,9 @@ try: bee_enabled = config['general'].getboolean('bee', False) # 🐝 off by default undocumented solar_conditions_enabled = config['general'].getboolean('spaceWeather', True) wikipedia_enabled = config['general'].getboolean('wikipedia', False) + use_kiwix_server = config['general'].getboolean('useKiwixServer', False) + kiwix_url = config['general'].get('kiwixURL', 'http://127.0.0.1:8080') + kiwix_library_name = config['general'].get('kiwixLibraryName', 'wikipedia_en_100_nopic_2024-06') llm_enabled = config['general'].getboolean('ollama', False) # https://ollama.com ollamaHostName = config['general'].get('ollamaHostName', 'http://localhost:11434') # default localhost llmModel = config['general'].get('ollamaModel', 'gemma3:270m') # default gemma3:270m diff --git a/modules/system.py b/modules/system.py index 71c0b46..67d60ab 100644 --- a/modules/system.py +++ b/modules/system.py @@ -209,6 +209,13 @@ if wikipedia_enabled: import wikipedia # pip install wikipedia trap_list = trap_list + ("wiki:", "wiki?",) help_message = help_message + ", wiki:" + + # Kiwix support for local wiki + if use_kiwix_server: + import requests + from bs4 import BeautifulSoup + from urllib.parse import quote + from bs4.element import Comment # LLM Configuration if llm_enabled: @@ -753,7 +760,86 @@ def send_message(message, ch, nodeid=0, nodeInt=1, bypassChuncking=False): interface.sendText(text=message, channelIndex=ch, destinationId=nodeid) return True +# Kiwix helper functions (only loaded if use_kiwix_server is True) +if wikipedia_enabled and use_kiwix_server: + def tag_visible(element): + """Filter visible text from HTML elements for Kiwix""" + if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']: + return False + if isinstance(element, Comment): + return False + return True + + def text_from_html(body): + """Extract visible text from HTML content""" + soup = BeautifulSoup(body, 'html.parser') + texts = soup.findAll(string=True) + visible_texts = filter(tag_visible, texts) + return " ".join(t.strip() for t in visible_texts if t.strip()) + + def get_kiwix_summary(search_term): + """Query local Kiwix server for Wikipedia article""" + try: + search_encoded = quote(search_term) + # Try direct article access first + wiki_article = search_encoded.capitalize().replace("%20", "_") + exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}" + + response = requests.get(exact_url, timeout=urlTimeoutSeconds) + if response.status_code == 200: + # Extract and clean text + text = text_from_html(response.text) + # Remove common Wikipedia metadata prefixes + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + # Truncate to reasonable length (first few sentences) + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] # Hard limit at 500 chars + + # If direct access fails, try search + search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" + response = requests.get(search_url, timeout=urlTimeoutSeconds) + + if response.status_code == 200 and "No results were found" not in response.text: + soup = BeautifulSoup(response.text, 'html.parser') + links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] + + for link in links[:3]: # Check first 3 results + article_name = link.split("/")[-1] + if not article_name or article_name[0].islower(): + continue + + article_url = f"{kiwix_url}{link}" + article_response = requests.get(article_url, timeout=urlTimeoutSeconds) + if article_response.status_code == 200: + text = text_from_html(article_response.text) + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] + + logger.warning(f"System: No Kiwix Results for:{search_term}") + return ERROR_FETCHING_DATA + + except requests.RequestException as e: + logger.warning(f"System: Kiwix connection error: {e}") + return "Unable to connect to local wiki server" + except Exception as e: + logger.warning(f"System: Error with Kiwix for:{search_term} {e}") + return ERROR_FETCHING_DATA + def get_wikipedia_summary(search_term): + # Use Kiwix if configured + if use_kiwix_server: + return get_kiwix_summary(search_term) + + # Otherwise use online Wikipedia wikipedia_search = wikipedia.search(search_term, results=3) wikipedia_suggest = wikipedia.suggest(search_term) #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) From 1e4e5e6627f47f99d55857c383c98b7a4962a479 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 8 Oct 2025 15:05:09 +0000 Subject: [PATCH 3/5] Add documentation and fix deprecated BeautifulSoup method Co-authored-by: SpudGunMan <12676665+SpudGunMan@users.noreply.github.com> --- README.md | 29 ++++++++++++++++++++++++++++- modules/system.py | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1cc4861..15e14f1 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ git clone https://github.com/spudgunman/meshing-around | `messages` | Replays the last messages heard on device, like Store and Forward, returns the PublicChannel and Current | ✅ | | `readnews` | returns the contents of a file (data/news.txt, by default) can also `news mesh` via the chunker on air | ✅ | | `satpass` | returns the pass info from API for defined NORAD ID in config or Example: `satpass 25544,33591`| | -| `wiki:` | Searches Wikipedia and returns the first few sentences of the first result if a match. Example: `wiki: lora radio` | +| `wiki:` | Searches Wikipedia (or local Kiwix server) and returns the first few sentences of the first result if a match. Example: `wiki: lora radio` | | `howfar` | returns the distance you have traveled since your last HowFar. `howfar reset` to start over | ✅ | | `howtall` | returns height of something you give a shadow by using sun angle | ✅ | @@ -397,6 +397,33 @@ googleSearchResults = 3 # number of google search results to include in the cont ``` Note for LLM in docker with [NVIDIA](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html). Needed for the container with ollama running. +### Wikipedia Search Settings +The Wikipedia search module can use either the online Wikipedia API or a local Kiwix server for offline wiki access. Kiwix is especially useful for mesh networks operating in remote or offline environments. + +```ini +# Enable or disable the wikipedia search module +wikipedia = True + +# Use local Kiwix server instead of online Wikipedia +# Set to False to use online Wikipedia (default) +useKiwixServer = False + +# Kiwix server URL (only used if useKiwixServer is True) +kiwixURL = http://127.0.0.1:8080 + +# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06) +# Find available libraries at https://library.kiwix.org/ +kiwixLibraryName = wikipedia_en_100_nopic_2024-06 +``` + +To set up a local Kiwix server: +1. Install Kiwix tools: https://kiwix.org/en/ +2. Download a Wikipedia ZIM file: https://library.kiwix.org/ +3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2024-06.zim` +4. Set `useKiwixServer = True` in your config.ini + +The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters). + ### Radio Monitoring A module allowing a Hamlib compatible radio to connect to the bot. When functioning, it will message the configured channel with a message of in use. **Requires hamlib/rigctld to be running as a service.** diff --git a/modules/system.py b/modules/system.py index 67d60ab..1a6dce0 100644 --- a/modules/system.py +++ b/modules/system.py @@ -773,7 +773,7 @@ if wikipedia_enabled and use_kiwix_server: def text_from_html(body): """Extract visible text from HTML content""" soup = BeautifulSoup(body, 'html.parser') - texts = soup.findAll(string=True) + texts = soup.find_all(string=True) visible_texts = filter(tag_visible, texts) return " ".join(t.strip() for t in visible_texts if t.strip()) From 413f2a24d9970086d1be15150df81147bb9dd64f Mon Sep 17 00:00:00 2001 From: SpudGunMan Date: Wed, 8 Oct 2025 20:29:52 -0700 Subject: [PATCH 4/5] Kiwix in wiki @NomDeTom its finally done --- README.md | 6 +-- config.template | 4 +- modules/system.py | 115 +------------------------------------------- modules/wiki.py | 119 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 118 deletions(-) create mode 100644 modules/wiki.py diff --git a/README.md b/README.md index 15e14f1..772cf57 100644 --- a/README.md +++ b/README.md @@ -417,9 +417,9 @@ kiwixLibraryName = wikipedia_en_100_nopic_2024-06 ``` To set up a local Kiwix server: -1. Install Kiwix tools: https://kiwix.org/en/ -2. Download a Wikipedia ZIM file: https://library.kiwix.org/ -3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2024-06.zim` +1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y` +2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim` +3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim` 4. Set `useKiwixServer = True` in your config.ini The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters). diff --git a/config.template b/config.template index 0e8a13c..421481f 100644 --- a/config.template +++ b/config.template @@ -62,8 +62,8 @@ wikipedia = True useKiwixServer = False # Kiwix server URL (e.g., http://127.0.0.1:8080) kiwixURL = http://127.0.0.1:8080 -# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06) -kiwixLibraryName = wikipedia_en_100_nopic_2024-06 +# Kiwix library name (e.g., wikipedia_en_100_nopic_2025-09) +kiwixLibraryName = wikipedia_en_100_nopic_2025-09 # Enable ollama LLM see more at https://ollama.com ollama = False diff --git a/modules/system.py b/modules/system.py index 1a6dce0..67811de 100644 --- a/modules/system.py +++ b/modules/system.py @@ -206,16 +206,9 @@ if dad_jokes_enabled: # Wikipedia Search Configuration if wikipedia_enabled: - import wikipedia # pip install wikipedia - trap_list = trap_list + ("wiki:", "wiki?",) + from modules.wiki import * # from the spudgunman/meshing-around repo + trap_list = trap_list + ("wiki:",) help_message = help_message + ", wiki:" - - # Kiwix support for local wiki - if use_kiwix_server: - import requests - from bs4 import BeautifulSoup - from urllib.parse import quote - from bs4.element import Comment # LLM Configuration if llm_enabled: @@ -760,110 +753,6 @@ def send_message(message, ch, nodeid=0, nodeInt=1, bypassChuncking=False): interface.sendText(text=message, channelIndex=ch, destinationId=nodeid) return True -# Kiwix helper functions (only loaded if use_kiwix_server is True) -if wikipedia_enabled and use_kiwix_server: - def tag_visible(element): - """Filter visible text from HTML elements for Kiwix""" - if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']: - return False - if isinstance(element, Comment): - return False - return True - - def text_from_html(body): - """Extract visible text from HTML content""" - soup = BeautifulSoup(body, 'html.parser') - texts = soup.find_all(string=True) - visible_texts = filter(tag_visible, texts) - return " ".join(t.strip() for t in visible_texts if t.strip()) - - def get_kiwix_summary(search_term): - """Query local Kiwix server for Wikipedia article""" - try: - search_encoded = quote(search_term) - # Try direct article access first - wiki_article = search_encoded.capitalize().replace("%20", "_") - exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}" - - response = requests.get(exact_url, timeout=urlTimeoutSeconds) - if response.status_code == 200: - # Extract and clean text - text = text_from_html(response.text) - # Remove common Wikipedia metadata prefixes - text = text.split("Jump to navigation", 1)[-1] - text = text.split("Jump to search", 1)[-1] - # Truncate to reasonable length (first few sentences) - sentences = text.split('. ') - summary = '. '.join(sentences[:wiki_return_limit]) - if summary and not summary.endswith('.'): - summary += '.' - return summary.strip()[:500] # Hard limit at 500 chars - - # If direct access fails, try search - search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" - response = requests.get(search_url, timeout=urlTimeoutSeconds) - - if response.status_code == 200 and "No results were found" not in response.text: - soup = BeautifulSoup(response.text, 'html.parser') - links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] - - for link in links[:3]: # Check first 3 results - article_name = link.split("/")[-1] - if not article_name or article_name[0].islower(): - continue - - article_url = f"{kiwix_url}{link}" - article_response = requests.get(article_url, timeout=urlTimeoutSeconds) - if article_response.status_code == 200: - text = text_from_html(article_response.text) - text = text.split("Jump to navigation", 1)[-1] - text = text.split("Jump to search", 1)[-1] - sentences = text.split('. ') - summary = '. '.join(sentences[:wiki_return_limit]) - if summary and not summary.endswith('.'): - summary += '.' - return summary.strip()[:500] - - logger.warning(f"System: No Kiwix Results for:{search_term}") - return ERROR_FETCHING_DATA - - except requests.RequestException as e: - logger.warning(f"System: Kiwix connection error: {e}") - return "Unable to connect to local wiki server" - except Exception as e: - logger.warning(f"System: Error with Kiwix for:{search_term} {e}") - return ERROR_FETCHING_DATA - -def get_wikipedia_summary(search_term): - # Use Kiwix if configured - if use_kiwix_server: - return get_kiwix_summary(search_term) - - # Otherwise use online Wikipedia - wikipedia_search = wikipedia.search(search_term, results=3) - wikipedia_suggest = wikipedia.suggest(search_term) - #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) - #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") - - if len(wikipedia_search) == 0: - logger.warning(f"System: No Wikipedia Results for:{search_term}") - return ERROR_FETCHING_DATA - - try: - logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}") - summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True) - except wikipedia.DisambiguationError as e: - logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}") - summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) - except wikipedia.PageError as e: - logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}") - summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) - except Exception as e: - logger.warning(f"System: Error with Wikipedia for:{search_term} {e}") - return ERROR_FETCHING_DATA - - return summary - def messageTrap(msg): # Check if the message contains a trap word, this is the first filter for listning to messages # after this the message is passed to the command_handler in the bot.py which is switch case filter for applying word to function diff --git a/modules/wiki.py b/modules/wiki.py new file mode 100644 index 0000000..ecfd8cf --- /dev/null +++ b/modules/wiki.py @@ -0,0 +1,119 @@ +# meshbot wiki module + +from modules.log import * +import wikipedia # pip install wikipedia + +# Kiwix support for local wiki +if use_kiwix_server: + import requests + from bs4 import BeautifulSoup + from urllib.parse import quote + from bs4.element import Comment + +# Kiwix helper functions (only loaded if use_kiwix_server is True) +if wikipedia_enabled and use_kiwix_server: + def tag_visible(element): + """Filter visible text from HTML elements for Kiwix""" + if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']: + return False + if isinstance(element, Comment): + return False + return True + + def text_from_html(body): + """Extract visible text from HTML content""" + soup = BeautifulSoup(body, 'html.parser') + texts = soup.find_all(string=True) + visible_texts = filter(tag_visible, texts) + return " ".join(t.strip() for t in visible_texts if t.strip()) + + def get_kiwix_summary(search_term): + """Query local Kiwix server for Wikipedia article""" + try: + search_encoded = quote(search_term) + # Try direct article access first + wiki_article = search_encoded.capitalize().replace("%20", "_") + exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}" + + response = requests.get(exact_url, timeout=urlTimeoutSeconds) + if response.status_code == 200: + # Extract and clean text + text = text_from_html(response.text) + # Remove common Wikipedia metadata prefixes + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + # Truncate to reasonable length (first few sentences) + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] # Hard limit at 500 chars + + # If direct access fails, try search + search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}" + response = requests.get(search_url, timeout=urlTimeoutSeconds) + + if response.status_code == 200 and "No results were found" not in response.text: + soup = BeautifulSoup(response.text, 'html.parser') + links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']] + + for link in links[:3]: # Check first 3 results + article_name = link.split("/")[-1] + if not article_name or article_name[0].islower(): + continue + + article_url = f"{kiwix_url}{link}" + article_response = requests.get(article_url, timeout=urlTimeoutSeconds) + if article_response.status_code == 200: + text = text_from_html(article_response.text) + text = text.split("Jump to navigation", 1)[-1] + text = text.split("Jump to search", 1)[-1] + sentences = text.split('. ') + summary = '. '.join(sentences[:wiki_return_limit]) + if summary and not summary.endswith('.'): + summary += '.' + return summary.strip()[:500] + + logger.warning(f"System: No Kiwix Results for:{search_term}") + return ERROR_FETCHING_DATA + + except requests.RequestException as e: + logger.warning(f"System: Kiwix connection error: {e}") + return "Unable to connect to local wiki server" + except Exception as e: + logger.warning(f"System: Error with Kiwix for:{search_term} {e}") + return ERROR_FETCHING_DATA + +def get_wikipedia_summary(search_term): + # Use Kiwix if configured + if use_kiwix_server: + return get_kiwix_summary(search_term) + + try: + # Otherwise use online Wikipedia + wikipedia_search = wikipedia.search(search_term, results=3) + wikipedia_suggest = wikipedia.suggest(search_term) + #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) + #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") + except Exception as e: + logger.debug(f"System: Wikipedia search error for:{search_term} {e}") + return ERROR_FETCHING_DATA + + if len(wikipedia_search) == 0: + logger.warning(f"System: No Wikipedia Results for:{search_term}") + return ERROR_FETCHING_DATA + + try: + logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}") + summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True) + except wikipedia.DisambiguationError as e: + logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}") + summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) + except wikipedia.PageError as e: + logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}") + summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True) + except Exception as e: + logger.warning(f"System: Error with Wikipedia for:{search_term} {e}") + return ERROR_FETCHING_DATA + + return summary \ No newline at end of file From 1aac3d5ac2c378bc73ed60e62ae4778c82c412b0 Mon Sep 17 00:00:00 2001 From: SpudGunMan Date: Wed, 8 Oct 2025 20:37:21 -0700 Subject: [PATCH 5/5] failover --- modules/wiki.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/wiki.py b/modules/wiki.py index ecfd8cf..4fdac0d 100644 --- a/modules/wiki.py +++ b/modules/wiki.py @@ -75,7 +75,9 @@ if wikipedia_enabled and use_kiwix_server: return summary.strip()[:500] logger.warning(f"System: No Kiwix Results for:{search_term}") - return ERROR_FETCHING_DATA + # try to fall back to online Wikipedia if available + return get_wikipedia_summary(search_term, force=True) + except requests.RequestException as e: logger.warning(f"System: Kiwix connection error: {e}") @@ -84,16 +86,17 @@ if wikipedia_enabled and use_kiwix_server: logger.warning(f"System: Error with Kiwix for:{search_term} {e}") return ERROR_FETCHING_DATA -def get_wikipedia_summary(search_term): +def get_wikipedia_summary(search_term, location=None, force=False): + lat, lon = location if location else (None, None) # Use Kiwix if configured - if use_kiwix_server: + if use_kiwix_server and not force: return get_kiwix_summary(search_term) try: # Otherwise use online Wikipedia wikipedia_search = wikipedia.search(search_term, results=3) wikipedia_suggest = wikipedia.suggest(search_term) - #wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3) + #wikipedia_aroundme = wikipedia.geosearch(lat,lon, results=3) #logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}") except Exception as e: logger.debug(f"System: Wikipedia search error for:{search_term} {e}")