From f02ac40b9129d1c12908adf74c51ad1c3ad42137 Mon Sep 17 00:00:00 2001 From: Joe Gitta Date: Wed, 27 May 2026 14:21:30 +0000 Subject: [PATCH] deploy: randomize and shuffle top RSS feeds --- browser.py | 126 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 51 deletions(-) diff --git a/browser.py b/browser.py index 367043b..56d663a 100755 --- a/browser.py +++ b/browser.py @@ -4,7 +4,7 @@ SeniorNet Kiosk — PyQt5 single-row floating toolbar over Google Chrome. Navigation: ◀ ▶ ⌂ icon buttons + ☰ hamburger menu for bookmarks. Chrome is controlled via Chrome DevTools Protocol (CDP) over WebSocket. """ -import sys, json, os, subprocess, time, threading, urllib.request, websocket, pathlib, shutil, sqlite3 +import sys, json, os, subprocess, time, threading, urllib.request, websocket, pathlib, shutil, sqlite3, random from PyQt5.QtCore import Qt, QTimer, QPoint, QRect, QEvent from PyQt5.QtGui import QFont, QColor, QPalette, QCursor from PyQt5.QtWidgets import ( @@ -864,15 +864,23 @@ if __name__ == "__main__": time.sleep(900) def update_news_cache(): - """Fetch top RSS news feeds (BBC, NPR), scrape full article text & images in background, and cache locally.""" + """Fetch top RSS news feeds, scrape full article text & images in background, and cache locally with randomized sources.""" import xml.etree.ElementTree as ET import re cache_file = _HOME / "kiosk-home" / "news-data.json" - feeds = { - "World News": ("http://feeds.bbci.co.uk/news/world/rss.xml", "BBC News"), - "National News": ("https://feeds.npr.org/1001/rss.xml", "NPR News") - } + # Expanded catalog of highly reliable news sources + world_catalog = [ + ("http://feeds.bbci.co.uk/news/world/rss.xml", "BBC News"), + ("https://rss.nytimes.com/services/xml/rss/nyt/World.xml", "New York Times"), + ("https://www.aljazeera.com/xml/rss/all.xml", "Al Jazeera") + ] + + national_catalog = [ + ("https://feeds.npr.org/1001/rss.xml", "NPR News"), + ("https://rss.nytimes.com/services/xml/rss/nyt/US.xml", "New York Times"), + ("http://rss.cnn.com/rss/cnn_topstories.rss", "CNN News") + ] def scrape_article(url): try: @@ -902,7 +910,8 @@ if __name__ == "__main__": ignore_terms = [ "British Broadcasting Corporation", "Watch Live", "HomeNewsSport", "Special Series", "Up First", "All Up First Stories", "hide caption", - "LISTEN & FOLLOW", "Morning Edition", "Morning news brief", "Podcast" + "LISTEN & FOLLOW", "Morning Edition", "Morning news brief", "Podcast", + "New York Times", "NYT", "The Times", "Al Jazeera", "BBC", "CNN" ] if any(term in p_clean for term in ignore_terms): continue @@ -916,56 +925,71 @@ if __name__ == "__main__": return "", "" while True: + # Randomly select 2 World feeds and 2 National feeds to harvest + selected_world = random.sample(world_catalog, 2) + selected_national = random.sample(national_catalog, 2) + + feeds = { + "World News": selected_world, + "National News": selected_national + } + payload = {} - for category, (url, source_name) in feeds.items(): - try: - req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - xml_raw = urllib.request.urlopen(req, timeout=12).read() - root = ET.fromstring(xml_raw) - - stories = [] - channel = root.find("channel") - if channel is not None: - # Scrape top 5 rich stories per category to keep it fast - for idx, item in enumerate(channel.findall("item")): - if idx >= 5: - break - title = item.find("title") - link = item.find("link") - pub_date = item.find("pubDate") - - t_str = title.text.strip() if title is not None and title.text else "" - l_str = link.text.strip() if link is not None and link.text else "" - - p_str = "" - if pub_date is not None and pub_date.text: - try: - p_str = " ".join(pub_date.text.split()[:4]) - except Exception: - p_str = pub_date.text - - # Perform background scraping for full text and image - full_story, img_url = "", "" - if l_str: - full_story, img_url = scrape_article(l_str) - - stories.append({ - "title": t_str, - "summary": full_story if full_story else "Click to read the story details.", - "image": img_url, - "source": source_name, - "date": p_str - }) - payload[category] = stories - print(f"Successfully cached and scraped {len(stories)} stories for category: {category}") - except Exception as e: - print(f"Failed to fetch RSS category {category}: {e}") + for category, source_list in feeds.items(): + category_stories = [] + for url, source_name in source_list: + try: + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + xml_raw = urllib.request.urlopen(req, timeout=12).read() + root = ET.fromstring(xml_raw) + + channel = root.find("channel") + if channel is not None: + # Fetch top 3 stories per selected feed to compile a rich, diverse set + items_harvested = 0 + for item in channel.findall("item"): + if items_harvested >= 3: + break + title = item.find("title") + link = item.find("link") + pub_date = item.find("pubDate") + + t_str = title.text.strip() if title is not None and title.text else "" + l_str = link.text.strip() if link is not None and link.text else "" + + p_str = "" + if pub_date is not None and pub_date.text: + try: + p_str = " ".join(pub_date.text.split()[:4]) + except Exception: + p_str = pub_date.text + + # Scrape rich data + full_story, img_url = "", "" + if l_str: + full_story, img_url = scrape_article(l_str) + + category_stories.append({ + "title": t_str, + "summary": full_story if full_story else "Click to read the story details.", + "image": img_url, + "source": source_name, + "date": p_str + }) + items_harvested += 1 + print(f"Harvested {items_harvested} stories from {source_name} for {category}") + except Exception as e: + print(f"Failed to harvest RSS feed {source_name} ({url}): {e}") + + # Fully randomize/shuffle the compiled stories list so sources are mixed nicely! + random.shuffle(category_stories) + payload[category] = category_stories if payload: try: cache_file.parent.mkdir(parents=True, exist_ok=True) cache_file.write_text(json.dumps(payload, indent=2)) - print("Updated local news-data.json cache with full stories successfully") + print("Updated local news-data.json cache with randomized shuffled stories successfully") except Exception as e: print(f"Failed to write news cache file: {e}")