deploy: randomize and shuffle top RSS feeds
This commit is contained in:
+42
-18
@@ -4,7 +4,7 @@ SeniorNet Kiosk — PyQt5 single-row floating toolbar over Google Chrome.
|
||||
Navigation: ◀ ▶ ⌂ icon buttons + ☰ hamburger menu for bookmarks.
|
||||
Chrome is controlled via Chrome DevTools Protocol (CDP) over WebSocket.
|
||||
"""
|
||||
import sys, json, os, subprocess, time, threading, urllib.request, websocket, pathlib, shutil, sqlite3
|
||||
import sys, json, os, subprocess, time, threading, urllib.request, websocket, pathlib, shutil, sqlite3, random
|
||||
from PyQt5.QtCore import Qt, QTimer, QPoint, QRect, QEvent
|
||||
from PyQt5.QtGui import QFont, QColor, QPalette, QCursor
|
||||
from PyQt5.QtWidgets import (
|
||||
@@ -864,15 +864,23 @@ if __name__ == "__main__":
|
||||
time.sleep(900)
|
||||
|
||||
def update_news_cache():
|
||||
"""Fetch top RSS news feeds (BBC, NPR), scrape full article text & images in background, and cache locally."""
|
||||
"""Fetch top RSS news feeds, scrape full article text & images in background, and cache locally with randomized sources."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import re
|
||||
cache_file = _HOME / "kiosk-home" / "news-data.json"
|
||||
|
||||
feeds = {
|
||||
"World News": ("http://feeds.bbci.co.uk/news/world/rss.xml", "BBC News"),
|
||||
"National News": ("https://feeds.npr.org/1001/rss.xml", "NPR News")
|
||||
}
|
||||
# Expanded catalog of highly reliable news sources
|
||||
world_catalog = [
|
||||
("http://feeds.bbci.co.uk/news/world/rss.xml", "BBC News"),
|
||||
("https://rss.nytimes.com/services/xml/rss/nyt/World.xml", "New York Times"),
|
||||
("https://www.aljazeera.com/xml/rss/all.xml", "Al Jazeera")
|
||||
]
|
||||
|
||||
national_catalog = [
|
||||
("https://feeds.npr.org/1001/rss.xml", "NPR News"),
|
||||
("https://rss.nytimes.com/services/xml/rss/nyt/US.xml", "New York Times"),
|
||||
("http://rss.cnn.com/rss/cnn_topstories.rss", "CNN News")
|
||||
]
|
||||
|
||||
def scrape_article(url):
|
||||
try:
|
||||
@@ -902,7 +910,8 @@ if __name__ == "__main__":
|
||||
ignore_terms = [
|
||||
"British Broadcasting Corporation", "Watch Live", "HomeNewsSport",
|
||||
"Special Series", "Up First", "All Up First Stories", "hide caption",
|
||||
"LISTEN & FOLLOW", "Morning Edition", "Morning news brief", "Podcast"
|
||||
"LISTEN & FOLLOW", "Morning Edition", "Morning news brief", "Podcast",
|
||||
"New York Times", "NYT", "The Times", "Al Jazeera", "BBC", "CNN"
|
||||
]
|
||||
if any(term in p_clean for term in ignore_terms):
|
||||
continue
|
||||
@@ -916,19 +925,30 @@ if __name__ == "__main__":
|
||||
return "", ""
|
||||
|
||||
while True:
|
||||
# Randomly select 2 World feeds and 2 National feeds to harvest
|
||||
selected_world = random.sample(world_catalog, 2)
|
||||
selected_national = random.sample(national_catalog, 2)
|
||||
|
||||
feeds = {
|
||||
"World News": selected_world,
|
||||
"National News": selected_national
|
||||
}
|
||||
|
||||
payload = {}
|
||||
for category, (url, source_name) in feeds.items():
|
||||
for category, source_list in feeds.items():
|
||||
category_stories = []
|
||||
for url, source_name in source_list:
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
xml_raw = urllib.request.urlopen(req, timeout=12).read()
|
||||
root = ET.fromstring(xml_raw)
|
||||
|
||||
stories = []
|
||||
channel = root.find("channel")
|
||||
if channel is not None:
|
||||
# Scrape top 5 rich stories per category to keep it fast
|
||||
for idx, item in enumerate(channel.findall("item")):
|
||||
if idx >= 5:
|
||||
# Fetch top 3 stories per selected feed to compile a rich, diverse set
|
||||
items_harvested = 0
|
||||
for item in channel.findall("item"):
|
||||
if items_harvested >= 3:
|
||||
break
|
||||
title = item.find("title")
|
||||
link = item.find("link")
|
||||
@@ -944,28 +964,32 @@ if __name__ == "__main__":
|
||||
except Exception:
|
||||
p_str = pub_date.text
|
||||
|
||||
# Perform background scraping for full text and image
|
||||
# Scrape rich data
|
||||
full_story, img_url = "", ""
|
||||
if l_str:
|
||||
full_story, img_url = scrape_article(l_str)
|
||||
|
||||
stories.append({
|
||||
category_stories.append({
|
||||
"title": t_str,
|
||||
"summary": full_story if full_story else "Click to read the story details.",
|
||||
"image": img_url,
|
||||
"source": source_name,
|
||||
"date": p_str
|
||||
})
|
||||
payload[category] = stories
|
||||
print(f"Successfully cached and scraped {len(stories)} stories for category: {category}")
|
||||
items_harvested += 1
|
||||
print(f"Harvested {items_harvested} stories from {source_name} for {category}")
|
||||
except Exception as e:
|
||||
print(f"Failed to fetch RSS category {category}: {e}")
|
||||
print(f"Failed to harvest RSS feed {source_name} ({url}): {e}")
|
||||
|
||||
# Fully randomize/shuffle the compiled stories list so sources are mixed nicely!
|
||||
random.shuffle(category_stories)
|
||||
payload[category] = category_stories
|
||||
|
||||
if payload:
|
||||
try:
|
||||
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache_file.write_text(json.dumps(payload, indent=2))
|
||||
print("Updated local news-data.json cache with full stories successfully")
|
||||
print("Updated local news-data.json cache with randomized shuffled stories successfully")
|
||||
except Exception as e:
|
||||
print(f"Failed to write news cache file: {e}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user