enhance

rss sucks

config.template

@@ -57,9 +57,9 @@ spaceWeather = True
 
 # enable or disable the RSS module, and truncate the story
 rssEnable = True
-rssFeedURL = http://www.hackaday.com/rss.xml,https://news.sparkfun.com/feeds/news
+rssFeedURL = http://www.hackaday.com/rss.xml,http://rss.slashdot.org/Slashdot/slashdotMain
 # RSS feed names must match the order of the URLs above, default is used if no match
-rssFeedNames = default,sparkfun
+rssFeedNames = default,slashdot
 rssMaxItems = 3
 rssTruncate = 100
 

modules/rss.py

@@ -2,6 +2,23 @@
 from modules.log import *
 import urllib.request
 import xml.etree.ElementTree as ET
+import html
+from html.parser import HTMLParser
+
+class MLStripper(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.reset()
+        self.fed = []
+    def handle_data(self, d):
+        self.fed.append(d)
+    def get_data(self):
+        return ''.join(self.fed)
+
+def strip_tags(html_text):
+    s = MLStripper()
+    s.feed(html_text)
+    return s.get_data()
 
 RSS_FEED_URLS = rssFeedURL
 RSS_FEED_NAMES = rssFeedNames
@@ -10,42 +27,69 @@ RSS_TRIM_LENGTH = rssTruncate
 
 def get_rss_feed(msg):
     # Determine which feed to use
-    feed_name = "default"
-    if msg and any(name in msg for name in RSS_FEED_NAMES):
+    feed_name = ""
+    msg_lower = msg.lower() if msg else ""
+    if msg_lower and any(name.lower() in msg_lower for name in RSS_FEED_NAMES):
         for name in RSS_FEED_NAMES:
-            if name in msg:
+            if name.lower() in msg_lower:
                 feed_name = name
                 break
+    else:
+        logger.debug(f"RSS: No feed name found in message '{msg}'. Using default feed.")
+        feed_name = RSS_FEED_NAMES[0] if RSS_FEED_NAMES else "default"
 
     try:
         idx = RSS_FEED_NAMES.index(feed_name)
         feed_url = RSS_FEED_URLS[idx]
     except (ValueError, IndexError):
+        logger.warning(f"RSS: Feed '{feed_name}' not found in RSS_FEED_URLS ({RSS_FEED_URLS}).")
         return f"Feed '{feed_name}' not found."
 
-    if "?" in msg:
-        return f"Fetches the latest {RSS_RETURN_COUNT} entries from the {feed_name} RSS feed."
+    if "?" in msg_lower:
+        return f"Fetches the latest {RSS_RETURN_COUNT} entries RSS feeds. Available feeds are: {', '.join(RSS_FEED_NAMES)}. To fetch a specific feed, include its name in your request."
 
     try:
-        with urllib.request.urlopen(feed_url, timeout= urlTimeoutSeconds) as response:
+        logger.debug(f"Fetching RSS feed from {feed_url} from message '{msg}'")
+        agent = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
+        request = urllib.request.Request(feed_url, headers=agent)
+        with urllib.request.urlopen(request, timeout=urlTimeoutSeconds) as response:
             xml_data = response.read()
         root = ET.fromstring(xml_data)
-        items = root.findall('.//item')[:RSS_RETURN_COUNT]
+        # Try both namespaced and non-namespaced item tags
+        items = root.findall('.//item')
+        ns = None
+        if not items:
+            # Try to find the namespace dynamically
+            for elem in root.iter():
+                if elem.tag.endswith('item'):
+                    ns_uri = elem.tag.split('}')[0].strip('{')
+                    items = root.findall(f'.//{{{ns_uri}}}item')
+                    ns = ns_uri
+                    break
+        items = items[:RSS_RETURN_COUNT]
         if not items:
             return "No RSS feed entries found."
         formatted_entries = []
         for item in items:
-            title = item.findtext('title', default='No title')
-            link = item.findtext('link', default='No link')
-            description = item.findtext('description', default='No description')
-            pub_date = item.findtext('pubDate', default='No date')
+            if ns:
+                title = item.findtext(f'{{{ns}}}title', default='No title')
+                link = item.findtext(f'{{{ns}}}link', default=None)
+                description = item.findtext(f'{{{ns}}}description', default='No description')
+                pub_date = item.findtext(f'{{{ns}}}pubDate', default='No date')
+            else:
+                title = item.findtext('title', default='No title')
+                link = item.findtext('link', default=None)
+                description = item.findtext('description', default='No description')
+                pub_date = item.findtext('pubDate', default='No date')
 
-            # strip all HTML tags and markup
-            description = ''.join(ET.fromstring(f"<div>{description}</div>").itertext())
+            # Unescape HTML entities and strip tags
+            description = html.unescape(description)
+            description = strip_tags(description)
             if len(description) > RSS_TRIM_LENGTH:
-                description = description[:97] + "..."
+                description = description[:RSS_TRIM_LENGTH - 3] + "..."
 
             formatted_entries.append(f"{title}\n{description}\n")
         return "\n".join(formatted_entries)
     except Exception as e:
+        logger.error(f"Error fetching RSS feed from {feed_url}: {e}")
         return ERROR_FETCHING_DATA