From 1cf63f40fec27a62bb1eaa09a0ac0cb2d18a767b Mon Sep 17 00:00:00 2001 From: Kelly Date: Tue, 18 Aug 2026 19:17:19 -0700 Subject: [PATCH] Update wiki.py Fixed Non-Prose Content Clutter Fixed Encoding/Mojibake Issue These changes ensure that when users run a local Kiwix server with useKiwixServer = True, they'll receive clean, properly-encoded article summaries https: //github.com/SpudGunMan/meshing-around/issues/334 Co-Authored-By: va3vbe <209848734+va3vbe@users.noreply.github.com> --- modules/wiki.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/wiki.py b/modules/wiki.py index 025321b..f1e194d 100644 --- a/modules/wiki.py +++ b/modules/wiki.py @@ -26,6 +26,21 @@ def text_from_html(body): main = soup.body if not main: return "" + + # Remove non-prose elements that pollute the summary: infoboxes, + # hatnotes/redirect notices, navigation boxes, citation markers, edit links. + for unwanted in main.select('table.infobox, table.sidebar, div.hatnote, div.navbox, div.metadata, sup.reference, .mw-editsection'): + unwanted.decompose() + + # Prefer extracting from

tags (actual lead paragraphs) + paragraphs = main.find_all('p') + if paragraphs: + texts = [p.get_text(" ", strip=True) for p in paragraphs] + texts = [t for t in texts if t] + if texts: + return " ".join(texts) + + # Fallback to existing full-text behavior if no

tags found texts = main.find_all(string=True) visible_texts = filter(tag_visible, texts) return " ".join(t.strip() for t in visible_texts if t.strip()) @@ -49,6 +64,8 @@ def get_kiwix_summary(search_term, truncate=True): continue article_url = f"{kiwix_url}{a['href']}" article_response = requests.get(article_url, timeout=urlTimeoutSeconds) + # Force UTF-8 encoding to avoid mojibake on multi-byte characters + article_response.encoding = 'utf-8' if article_response.status_code == 200: text = text_from_html(article_response.text) # Remove navigation and search jump text