deploy: filter global navs from RSS news cache

2026-05-27 14:12:36 +00:00
parent 6a4eee9a15
commit 598bd83887
1 changed files with 8 additions and 0 deletions
@@ -898,6 +898,14 @@ if __name__ == "__main__":
                    
                    # Filter out short noise paragraphs (ads, nav links)
                    if len(p_clean) > 85 and not p_clean.startswith("Follow ") and not p_clean.startswith("Read more") and not "click here" in p_clean.lower() and not "twitter" in p_clean.lower():
+                        # Standard library blocklist to discard site global header/footer navigation items
+                        ignore_terms = [
+                            "British Broadcasting Corporation", "Watch Live", "HomeNewsSport", 
+                            "Special Series", "Up First", "All Up First Stories", "hide caption", 
+                            "LISTEN & FOLLOW", "Morning Edition", "Morning news brief", "Podcast"
+                        ]
+                        if any(term in p_clean for term in ignore_terms):
+                            continue
                        paragraphs.append(p_clean)
                
                # Keep top 8 substantial paragraphs