From d608754b5ee653ed981d4fb66310bd0b678386da Mon Sep 17 00:00:00 2001 From: SpudGunMan Date: Sun, 26 Oct 2025 21:51:02 -0700 Subject: [PATCH] dedupe --- modules/rss.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/rss.py b/modules/rss.py index fdded3c..2ad64c6 100644 --- a/modules/rss.py +++ b/modules/rss.py @@ -77,6 +77,7 @@ def get_rss_feed(msg): return "No RSS or Atom feed entries found." formatted_entries = [] + seen_first3 = set() # Track first 3 words (lowercased) to avoid duplicates for item in items: # Helper to try multiple tag names def find_any(item, tags): @@ -122,9 +123,16 @@ def get_rss_feed(msg): if len(description) > RSS_TRIM_LENGTH: description = description[:RSS_TRIM_LENGTH - 3] + "..." + # Duplicate check: use first 3 words of description (or title if description is empty) + text_for_dupe = description if description else (title or "") + first3 = " ".join(text_for_dupe.lower().split()[:3]) + if first3 in seen_first3: + continue + seen_first3.add(first3) + formatted_entries.append(f"{title}\n{description}\n") return "\n".join(formatted_entries) except Exception as e: logger.error(f"Error fetching RSS feed from {feed_url}: {e}") return ERROR_FETCHING_DATA - \ No newline at end of file +