adam/meshing-around
1
0
Fork 0
mirror of https://github.com/SpudGunMan/meshing-around.git synced 2026-08-04 16:02:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

Kiwix in wiki

Browse Source 
@NomDeTom its finally done
This commit is contained in:
SpudGunMan
2025-10-08 20:29:52 -07:00
parent 1e4e5e6627
commit 413f2a24d9
4 changed files with 126 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -3
View File
@@ -417,9 +417,9 @@ kiwixLibraryName = wikipedia_en_100_nopic_2024-06
```
To set up a local Kiwix server:
1. Install Kiwix tools: https://kiwix.org/en/
2. Download a Wikipedia ZIM file: https://library.kiwix.org/
3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2024-06.zim`
1. Install Kiwix tools: https://kiwix.org/en/ `sudo apt install kiwix-tools -y`
2. Download a Wikipedia ZIM file to `data/`: https://library.kiwix.org/ `wget https://download.kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2025-09.zim`
3. Run the server: `kiwix-serve --port 8080 wikipedia_en_100_nopic_2025-09.zim`
4. Set `useKiwixServer = True` in your config.ini
The bot will automatically extract and truncate content to fit Meshtastic's message size limits (~500 characters).
config.template
+2 -2
View File
@@ -62,8 +62,8 @@ wikipedia = True
useKiwixServer = False
# Kiwix server URL (e.g., http://127.0.0.1:8080)
kiwixURL = http://127.0.0.1:8080
# Kiwix library name (e.g., wikipedia_en_100_nopic_2024-06)
kiwixLibraryName = wikipedia_en_100_nopic_2024-06
# Kiwix library name (e.g., wikipedia_en_100_nopic_2025-09)
kiwixLibraryName = wikipedia_en_100_nopic_2025-09
# Enable ollama LLM see more at https://ollama.com
ollama = False
modules/system.py
+2 -113
View File
@@ -206,16 +206,9 @@ if dad_jokes_enabled:
# Wikipedia Search Configuration
if wikipedia_enabled:
import wikipedia # pip install wikipedia
trap_list = trap_list + ("wiki:", "wiki?",)
from modules.wiki import * # from the spudgunman/meshing-around repo
trap_list = trap_list + ("wiki:",)
help_message = help_message + ", wiki:"
# Kiwix support for local wiki
if use_kiwix_server:
import requests
from bs4 import BeautifulSoup
from urllib.parse import quote
from bs4.element import Comment
# LLM Configuration
if llm_enabled:
@@ -760,110 +753,6 @@ def send_message(message, ch, nodeid=0, nodeInt=1, bypassChuncking=False):
interface.sendText(text=message, channelIndex=ch, destinationId=nodeid)
return True
# Kiwix helper functions (only loaded if use_kiwix_server is True)
if wikipedia_enabled and use_kiwix_server:
def tag_visible(element):
"""Filter visible text from HTML elements for Kiwix"""
if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
return False
if isinstance(element, Comment):
return False
return True
def text_from_html(body):
"""Extract visible text from HTML content"""
soup = BeautifulSoup(body, 'html.parser')
texts = soup.find_all(string=True)
visible_texts = filter(tag_visible, texts)
return " ".join(t.strip() for t in visible_texts if t.strip())
def get_kiwix_summary(search_term):
"""Query local Kiwix server for Wikipedia article"""
try:
search_encoded = quote(search_term)
# Try direct article access first
wiki_article = search_encoded.capitalize().replace("%20", "_")
exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}"
response = requests.get(exact_url, timeout=urlTimeoutSeconds)
if response.status_code == 200:
# Extract and clean text
text = text_from_html(response.text)
# Remove common Wikipedia metadata prefixes
text = text.split("Jump to navigation", 1)[-1]
text = text.split("Jump to search", 1)[-1]
# Truncate to reasonable length (first few sentences)
sentences = text.split('. ')
summary = '. '.join(sentences[:wiki_return_limit])
if summary and not summary.endswith('.'):
summary += '.'
return summary.strip()[:500] # Hard limit at 500 chars
# If direct access fails, try search
search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}"
response = requests.get(search_url, timeout=urlTimeoutSeconds)
if response.status_code == 200 and "No results were found" not in response.text:
soup = BeautifulSoup(response.text, 'html.parser')
links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']]
for link in links[:3]: # Check first 3 results
article_name = link.split("/")[-1]
if not article_name or article_name[0].islower():
continue
article_url = f"{kiwix_url}{link}"
article_response = requests.get(article_url, timeout=urlTimeoutSeconds)
if article_response.status_code == 200:
text = text_from_html(article_response.text)
text = text.split("Jump to navigation", 1)[-1]
text = text.split("Jump to search", 1)[-1]
sentences = text.split('. ')
summary = '. '.join(sentences[:wiki_return_limit])
if summary and not summary.endswith('.'):
summary += '.'
return summary.strip()[:500]
logger.warning(f"System: No Kiwix Results for:{search_term}")
return ERROR_FETCHING_DATA
except requests.RequestException as e:
logger.warning(f"System: Kiwix connection error: {e}")
return "Unable to connect to local wiki server"
except Exception as e:
logger.warning(f"System: Error with Kiwix for:{search_term} {e}")
return ERROR_FETCHING_DATA
def get_wikipedia_summary(search_term):
# Use Kiwix if configured
if use_kiwix_server:
return get_kiwix_summary(search_term)
# Otherwise use online Wikipedia
wikipedia_search = wikipedia.search(search_term, results=3)
wikipedia_suggest = wikipedia.suggest(search_term)
#wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3)
#logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}")
if len(wikipedia_search) == 0:
logger.warning(f"System: No Wikipedia Results for:{search_term}")
return ERROR_FETCHING_DATA
try:
logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}")
summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True)
except wikipedia.DisambiguationError as e:
logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}")
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
except wikipedia.PageError as e:
logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}")
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
except Exception as e:
logger.warning(f"System: Error with Wikipedia for:{search_term} {e}")
return ERROR_FETCHING_DATA
return summary
def messageTrap(msg):
# Check if the message contains a trap word, this is the first filter for listning to messages
# after this the message is passed to the command_handler in the bot.py which is switch case filter for applying word to function
modules/wiki.py
+119
View File
@@ -0,0 +1,119 @@
# meshbot wiki module
from modules.log import *
import wikipedia # pip install wikipedia
# Kiwix support for local wiki
if use_kiwix_server:
import requests
from bs4 import BeautifulSoup
from urllib.parse import quote
from bs4.element import Comment
# Kiwix helper functions (only loaded if use_kiwix_server is True)
if wikipedia_enabled and use_kiwix_server:
def tag_visible(element):
"""Filter visible text from HTML elements for Kiwix"""
if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
return False
if isinstance(element, Comment):
return False
return True
def text_from_html(body):
"""Extract visible text from HTML content"""
soup = BeautifulSoup(body, 'html.parser')
texts = soup.find_all(string=True)
visible_texts = filter(tag_visible, texts)
return " ".join(t.strip() for t in visible_texts if t.strip())
def get_kiwix_summary(search_term):
"""Query local Kiwix server for Wikipedia article"""
try:
search_encoded = quote(search_term)
# Try direct article access first
wiki_article = search_encoded.capitalize().replace("%20", "_")
exact_url = f"{kiwix_url}/raw/{kiwix_library_name}/content/A/{wiki_article}"
response = requests.get(exact_url, timeout=urlTimeoutSeconds)
if response.status_code == 200:
# Extract and clean text
text = text_from_html(response.text)
# Remove common Wikipedia metadata prefixes
text = text.split("Jump to navigation", 1)[-1]
text = text.split("Jump to search", 1)[-1]
# Truncate to reasonable length (first few sentences)
sentences = text.split('. ')
summary = '. '.join(sentences[:wiki_return_limit])
if summary and not summary.endswith('.'):
summary += '.'
return summary.strip()[:500] # Hard limit at 500 chars
# If direct access fails, try search
search_url = f"{kiwix_url}/search?content={kiwix_library_name}&pattern={search_encoded}"
response = requests.get(search_url, timeout=urlTimeoutSeconds)
if response.status_code == 200 and "No results were found" not in response.text:
soup = BeautifulSoup(response.text, 'html.parser')
links = [a['href'] for a in soup.find_all('a', href=True) if "start=" not in a['href']]
for link in links[:3]: # Check first 3 results
article_name = link.split("/")[-1]
if not article_name or article_name[0].islower():
continue
article_url = f"{kiwix_url}{link}"
article_response = requests.get(article_url, timeout=urlTimeoutSeconds)
if article_response.status_code == 200:
text = text_from_html(article_response.text)
text = text.split("Jump to navigation", 1)[-1]
text = text.split("Jump to search", 1)[-1]
sentences = text.split('. ')
summary = '. '.join(sentences[:wiki_return_limit])
if summary and not summary.endswith('.'):
summary += '.'
return summary.strip()[:500]
logger.warning(f"System: No Kiwix Results for:{search_term}")
return ERROR_FETCHING_DATA
except requests.RequestException as e:
logger.warning(f"System: Kiwix connection error: {e}")
return "Unable to connect to local wiki server"
except Exception as e:
logger.warning(f"System: Error with Kiwix for:{search_term} {e}")
return ERROR_FETCHING_DATA
def get_wikipedia_summary(search_term):
# Use Kiwix if configured
if use_kiwix_server:
return get_kiwix_summary(search_term)
try:
# Otherwise use online Wikipedia
wikipedia_search = wikipedia.search(search_term, results=3)
wikipedia_suggest = wikipedia.suggest(search_term)
#wikipedia_aroundme = wikipedia.geosearch(location[0], location[1], results=3)
#logger.debug(f"System: Wikipedia Nearby:{wikipedia_aroundme}")
except Exception as e:
logger.debug(f"System: Wikipedia search error for:{search_term} {e}")
return ERROR_FETCHING_DATA
if len(wikipedia_search) == 0:
logger.warning(f"System: No Wikipedia Results for:{search_term}")
return ERROR_FETCHING_DATA
try:
logger.debug(f"System: Searching Wikipedia for:{search_term}, First Result:{wikipedia_search[0]}, Suggest Word:{wikipedia_suggest}")
summary = wikipedia.summary(search_term, sentences=wiki_return_limit, auto_suggest=False, redirect=True)
except wikipedia.DisambiguationError as e:
logger.warning(f"System: Disambiguation Error for:{search_term} trying {wikipedia_search[0]}")
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
except wikipedia.PageError as e:
logger.warning(f"System: Wikipedia Page Error for:{search_term} {e} trying {wikipedia_search[0]}")
summary = wikipedia.summary(wikipedia_search[0], sentences=wiki_return_limit, auto_suggest=True, redirect=True)
except Exception as e:
logger.warning(f"System: Error with Wikipedia for:{search_term} {e}")
return ERROR_FETCHING_DATA
return summary