jgitta/senior-kiosk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: upgrade local newspaper to full-text and images

Browse Source
This commit is contained in:
Jozsef Gtta
2026-05-27 14:12:08 +00:00
parent 4b1ea2da11
commit 6a4eee9a15
2 changed files with 56 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser.py
+48 -12
View File
@@ -864,7 +864,7 @@ if __name__ == "__main__":
time.sleep(900)
def update_news_cache():
"""Fetch top RSS news feeds (BBC, NPR) in background and cache as local JSON to bypass CORS/Ads."""
"""Fetch top RSS news feeds (BBC, NPR), scrape full article text & images in background, and cache locally."""
import xml.etree.ElementTree as ET
import re
cache_file = _HOME / "kiosk-home" / "news-data.json"
@@ -874,6 +874,39 @@ if __name__ == "__main__":
"National News": "https://feeds.npr.org/1001/rss.xml"
}
def scrape_article(url):
try:
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
html = urllib.request.urlopen(req, timeout=10).read().decode('utf-8', errors='ignore')
# 1. Extract og:image
img_url = ""
og_img_match = re.search(r'<meta[^>]*property=["\']og:image["\'][^>]*content=["\']([^"\']+)["\']', html)
if not og_img_match:
og_img_match = re.search(r'<meta[^>]*content=["\']([^"\']+)["\'][^>]*property=["\']og:image["\']', html)
if og_img_match:
img_url = og_img_match.group(1)
# 2. Extract main paragraphs
paragraphs = []
p_matches = re.findall(r'<p[^>]*>(.*?)</p>', html, re.DOTALL)
for p in p_matches:
p_clean = re.sub('<[^<]+?>', '', p)
p_clean = p_clean.strip()
# Decode basic HTML entities
p_clean = p_clean.replace("&nbsp;", " ").replace("&amp;", "&").replace("&quot;", '"').replace("&apos;", "'").replace("&#39;", "'").replace("&ndash;", "-").replace("&mdash;", "-")
# Filter out short noise paragraphs (ads, nav links)
if len(p_clean) > 85 and not p_clean.startswith("Follow ") and not p_clean.startswith("Read more") and not "click here" in p_clean.lower() and not "twitter" in p_clean.lower():
paragraphs.append(p_clean)
# Keep top 8 substantial paragraphs
full_text = "\n\n".join(paragraphs[:8])
return full_text, img_url
except Exception as e:
print(f"Scraping error for {url}: {e}")
return "", ""
while True:
payload = {}
for category, url in feeds.items():
@@ -885,19 +918,16 @@ if __name__ == "__main__":
stories = []
channel = root.find("channel")
if channel is not None:
# Fetch up to 10 stories per category
# Scrape top 5 rich stories per category to keep it fast
for idx, item in enumerate(channel.findall("item")):
if idx >= 10:
if idx >= 5:
break
title = item.find("title")
desc = item.find("description")
link = item.find("link")
pub_date = item.find("pubDate")
t_str = title.text.strip() if title is not None and title.text else ""
d_str = desc.text.strip() if desc is not None and desc.text else ""
# Clean HTML tags from description if any
d_str = re.sub('<[^<]+?>', '', d_str)
l_str = link.text.strip() if link is not None and link.text else ""
p_str = ""
if pub_date is not None and pub_date.text:
@@ -905,14 +935,20 @@ if __name__ == "__main__":
p_str = " ".join(pub_date.text.split()[:4])
except Exception:
p_str = pub_date.text
# Perform background scraping for full text and image
full_story, img_url = "", ""
if l_str:
full_story, img_url = scrape_article(l_str)
stories.append({
"title": t_str,
"summary": d_str,
"summary": full_story if full_story else "Click to read the story details.",
"image": img_url,
"date": p_str
})
payload[category] = stories
print(f"Successfully cached {len(stories)} stories for category: {category}")
print(f"Successfully cached and scraped {len(stories)} stories for category: {category}")
except Exception as e:
print(f"Failed to fetch RSS category {category}: {e}")
@@ -920,7 +956,7 @@ if __name__ == "__main__":
try:
cache_file.parent.mkdir(parents=True, exist_ok=True)
cache_file.write_text(json.dumps(payload, indent=2))
print("Updated local news-data.json cache successfully")
print("Updated local news-data.json cache with full stories successfully")
except Exception as e:
print(f"Failed to write news cache file: {e}")