mirror of
https://github.com/SpudGunMan/meshing-around.git
synced 2026-07-01 23:41:14 +02:00
Merge pull request #203 from SpudGunMan/copilot/add-local-wiki-search-functionality
Add Kiwix local wiki server support for offline Wikipedia searches
This commit is contained in:
Kelly
@@ -150,7 +150,7 @@ git clone https://github.com/spudgunman/meshing-around
|
||||
| `messages` | Replays the last messages heard on device, like Store and Forward, returns the PublicChannel and Current | ✅ |
|
||||
| `readnews` | returns the contents of a file (data/news.txt, by default) can also `news mesh` via the chunker on air | ✅ |
|
||||
| `satpass` | returns the pass info from API for defined NORAD ID in config or Example: `satpass 25544,33591`| |
|
||||
| `wiki:` | Searches Wikipedia and returns the first few sentences of the first result if a match. Example: `wiki: lora radio` |
|
||||
| `wiki:` | Searches Wikipedia (or local Kiwix server) and returns the first few sentences of the first result if a match. Example: `wiki: lora radio` |
|
||||
| `howfar` | returns the distance you have traveled since your last HowFar. `howfar reset` to start over | ✅ |
|
||||
| `howtall` | returns height of something you give a shadow by using sun angle | ✅ |
|
||||
|
||||
@@ -398,6 +398,33 @@ googleSearchResults = 3 # number of google search results to include in the cont
|
||||
```
|
||||
Note for LLM in docker with [NVIDIA](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html). Needed for the container with ollama running.
|
||||
|
||||
### Wikipedia Search Settings
|
||||
The Wikipedia search module can use either the online Wikipedia API or a local Kiwix server for offline wiki access. Kiwix is especially useful for mesh networks operating in remote or offline environments.
|
||||
|
||||
```ini
|
||||
# Enable or disable the wikipedia search module
|
||||
wikipedia = True
|
||||
|
||||
# Use local Kiwix server instead of online Wikipedia
|
||||
# Set to False to use online Wikipedia (default)
|
||||
useKiwixServer = False
|
||||
|
||||
# Kiwix server URL (only used if useKiwixServer is True)
|
||||
kiwixURL = http://127.0.0.1:8080
|
||||
|
||||
# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06)
|
||||
# Find available libraries at https://library.kiwix.org/
|
||||
kiwixLibraryName = wikipedia_en_100_nopic_2024-06
|
||||
```
|
||||
|
||||
To set up a local Kiwix server:
|
||||
1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y`
|
||||
2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim`
|
||||
3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim`
|
||||
4. Set `useKiwixServer = True` in your config.ini
|
||||
|
||||
The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters).
|
||||
|
||||
### Radio Monitoring
|
||||
A module allowing a Hamlib compatible radio to connect to the bot. When functioning, it will message the configured channel with a message of in use. **Requires hamlib/rigctld to be running as a service.**
|
||||
|
||||
|
||||
@@ -57,6 +57,13 @@ spaceWeather = True
|
||||
|
||||
# enable or disable the wikipedia search module
|
||||
wikipedia = True
|
||||
# Use local Kiwix server instead of online Wikipedia
|
||||
# Set to False to use online Wikipedia, or provide Kiwix server URL
|
||||
useKiwixServer = False
|
||||
# Kiwix server URL (e.g., http://127.0.0.1:8080)
|
||||
kiwixURL = http://127.0.0.1:8080
|
||||
# Kiwix library name (e.g., wikipedia_en_100_nopic_2025-09)
|
||||
kiwixLibraryName = wikipedia_en_100_nopic_2025-09
|
||||
|
||||
# Enable ollama LLM see more at https://ollama.com
|
||||
ollama = False
|
||||
|
||||
@@ -227,6 +227,9 @@ try:
|
||||
bee_enabled = config['general'].getboolean('bee', False) # 🐝 off by default undocumented
|
||||
solar_conditions_enabled = config['general'].getboolean('spaceWeather', True)
|
||||
wikipedia_enabled = config['general'].getboolean('wikipedia', False)
|
||||
use_kiwix_server = config['general'].getboolean('useKiwixServer', False)
|
||||
kiwix_url = config['general'].get('kiwixURL', 'http://127.0.0.1:8080')
|
||||
kiwix_library_name = config['general'].get('kiwixLibraryName', 'wikipedia_en_100_nopic_2024-06')
|
||||
llm_enabled = config['general'].getboolean('ollama', False) # https://ollama.com
|
||||
ollamaHostName = config['general'].get('ollamaHostName', 'http://localhost:11434') # default localhost
|
||||
llmModel = config['general'].get('ollamaModel', 'gemma3:270m') # default gemma3:270m
|
||||
|
||||
+2
-27
@@ -206,8 +206,8 @@ if dad_jokes_enabled:
|
||||
|
||||
# Wikipedia Search Configuration
|
||||
if wikipedia_enabled:
|
||||
import wikipedia # pip install wikipedia
|
||||
trap_list = trap_list + ("wiki:", "wiki?",)
|
||||
from modules.wiki import * # from the spudgunman/meshing-around repo
|
||||
trap_list = trap_list + ("wiki:",)
|
||||
help_message = help_message + ", wiki:"
|
||||
|
||||
# LLM Configuration
|
||||
@@ -753,31 +753,6 @@ def send_message(message, ch, nodeid=0, nodeInt=1, bypassChuncking=False):
|
||||
interface.sendText(text=message, channelIndex=ch, destinationId=nodeid)
|
||||
return True
|
||||
|
||||
def get_wikipedia_summary(search_term):
|
||||
wikipedia_search = wikipedia.search(search_term, results=3)
|
||||
wikipedia_suggest = wikipedia.suggest(search_term)
|
||||
#wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3)
|
||||
#logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}")
|
||||
|
||||
if len(wikipedia_search) == 0:
|
||||
logger.warning(f"System: No Wikipedia Results for:{search_term}")
|
||||
return ERROR_FETCHING_DATA
|
||||
|
||||
try:
|
||||
logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}")
|
||||
summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True)
|
||||
except wikipedia.DisambiguationError as e:
|
||||
logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}")
|
||||
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
|
||||
except wikipedia.PageError as e:
|
||||
logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}")
|
||||
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"System: Error with Wikipedia for:{search_term} {e}")
|
||||
return ERROR_FETCHING_DATA
|
||||
|
||||
return summary
|
||||
|
||||
def messageTrap(msg):
|
||||
# Check if the message contains a trap word, this is the first filter for listning to messages
|
||||
# after this the message is passed to the command_handler in the bot.py which is switch case filter for applying word to function
|
||||
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
# meshbot wiki module
|
||||
|
||||
from modules.log import *
|
||||
import wikipedia # pip install wikipedia
|
||||
|
||||
# Kiwix support for local wiki
|
||||
if use_kiwix_server:
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import quote
|
||||
from bs4.element import Comment
|
||||
|
||||
# Kiwix helper functions (only loaded if use_kiwix_server is True)
|
||||
if wikipedia_enabled and use_kiwix_server:
|
||||
def tag_visible(element):
|
||||
"""Filter visible text from HTML elements for Kiwix"""
|
||||
if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
|
||||
return False
|
||||
if isinstance(element, Comment):
|
||||
return False
|
||||
return True
|
||||
|
||||
def text_from_html(body):
|
||||
"""Extract visible text from HTML content"""
|
||||
soup = BeautifulSoup(body, 'html.parser')
|
||||
texts = soup.find_all(string=True)
|
||||
visible_texts = filter(tag_visible, texts)
|
||||
return " ".join(t.strip() for t in visible_texts if t.strip())
|
||||
|
||||
def get_kiwix_summary(search_term):
|
||||
"""Query local Kiwix server for Wikipedia article"""
|
||||
try:
|
||||
search_encoded = quote(search_term)
|
||||
# Try direct article access first
|
||||
wiki_article = search_encoded.capitalize().replace("%20", "_")
|
||||
exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}"
|
||||
|
||||
response = requests.get(exact_url, timeout=urlTimeoutSeconds)
|
||||
if response.status_code == 200:
|
||||
# Extract and clean text
|
||||
text = text_from_html(response.text)
|
||||
# Remove common Wikipedia metadata prefixes
|
||||
text = text.split("Jump to navigation", 1)[-1]
|
||||
text = text.split("Jump to search", 1)[-1]
|
||||
# Truncate to reasonable length (first few sentences)
|
||||
sentences = text.split('. ')
|
||||
summary = '. '.join(sentences[:wiki_return_limit])
|
||||
if summary and not summary.endswith('.'):
|
||||
summary += '.'
|
||||
return summary.strip()[:500] # Hard limit at 500 chars
|
||||
|
||||
# If direct access fails, try search
|
||||
search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}"
|
||||
response = requests.get(search_url, timeout=urlTimeoutSeconds)
|
||||
|
||||
if response.status_code == 200 and "No results were found" not in response.text:
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']]
|
||||
|
||||
for link in links[:3]: # Check first 3 results
|
||||
article_name = link.split("/")[-1]
|
||||
if not article_name or article_name[0].islower():
|
||||
continue
|
||||
|
||||
article_url = f"{kiwix_url}{link}"
|
||||
article_response = requests.get(article_url, timeout=urlTimeoutSeconds)
|
||||
if article_response.status_code == 200:
|
||||
text = text_from_html(article_response.text)
|
||||
text = text.split("Jump to navigation", 1)[-1]
|
||||
text = text.split("Jump to search", 1)[-1]
|
||||
sentences = text.split('. ')
|
||||
summary = '. '.join(sentences[:wiki_return_limit])
|
||||
if summary and not summary.endswith('.'):
|
||||
summary += '.'
|
||||
return summary.strip()[:500]
|
||||
|
||||
logger.warning(f"System: No Kiwix Results for:{search_term}")
|
||||
# try to fall back to online Wikipedia if available
|
||||
return get_wikipedia_summary(search_term, force=True)
|
||||
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.warning(f"System: Kiwix connection error: {e}")
|
||||
return "Unable to connect to local wiki server"
|
||||
except Exception as e:
|
||||
logger.warning(f"System: Error with Kiwix for:{search_term} {e}")
|
||||
return ERROR_FETCHING_DATA
|
||||
|
||||
def get_wikipedia_summary(search_term, location=None, force=False):
|
||||
lat, lon = location if location else (None, None)
|
||||
# Use Kiwix if configured
|
||||
if use_kiwix_server and not force:
|
||||
return get_kiwix_summary(search_term)
|
||||
|
||||
try:
|
||||
# Otherwise use online Wikipedia
|
||||
wikipedia_search = wikipedia.search(search_term, results=3)
|
||||
wikipedia_suggest = wikipedia.suggest(search_term)
|
||||
#wikipedia_aroundme = wikipedia.geosearch(lat,lon, results=3)
|
||||
#logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}")
|
||||
except Exception as e:
|
||||
logger.debug(f"System: Wikipedia search error for:{search_term} {e}")
|
||||
return ERROR_FETCHING_DATA
|
||||
|
||||
if len(wikipedia_search) == 0:
|
||||
logger.warning(f"System: No Wikipedia Results for:{search_term}")
|
||||
return ERROR_FETCHING_DATA
|
||||
|
||||
try:
|
||||
logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}")
|
||||
summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True)
|
||||
except wikipedia.DisambiguationError as e:
|
||||
logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}")
|
||||
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
|
||||
except wikipedia.PageError as e:
|
||||
logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}")
|
||||
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"System: Error with Wikipedia for:{search_term} {e}")
|
||||
return ERROR_FETCHING_DATA
|
||||
|
||||
return summary
|
||||
Reference in New Issue
Block a user