diff --git a/browser.py b/browser.py index 9398472..97c0fec 100755 --- a/browser.py +++ b/browser.py @@ -864,7 +864,7 @@ if __name__ == "__main__": time.sleep(900) def update_news_cache(): - """Fetch top RSS news feeds (BBC, NPR) in background and cache as local JSON to bypass CORS/Ads.""" + """Fetch top RSS news feeds (BBC, NPR), scrape full article text & images in background, and cache locally.""" import xml.etree.ElementTree as ET import re cache_file = _HOME / "kiosk-home" / "news-data.json" @@ -874,6 +874,39 @@ if __name__ == "__main__": "National News": "https://feeds.npr.org/1001/rss.xml" } + def scrape_article(url): + try: + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + html = urllib.request.urlopen(req, timeout=10).read().decode('utf-8', errors='ignore') + + # 1. Extract og:image + img_url = "" + og_img_match = re.search(r']*property=["\']og:image["\'][^>]*content=["\']([^"\']+)["\']', html) + if not og_img_match: + og_img_match = re.search(r']*content=["\']([^"\']+)["\'][^>]*property=["\']og:image["\']', html) + if og_img_match: + img_url = og_img_match.group(1) + + # 2. Extract main paragraphs + paragraphs = [] + p_matches = re.findall(r']*>(.*?)

', html, re.DOTALL) + for p in p_matches: + p_clean = re.sub('<[^<]+?>', '', p) + p_clean = p_clean.strip() + # Decode basic HTML entities + p_clean = p_clean.replace(" ", " ").replace("&", "&").replace(""", '"').replace("'", "'").replace("'", "'").replace("–", "-").replace("—", "-") + + # Filter out short noise paragraphs (ads, nav links) + if len(p_clean) > 85 and not p_clean.startswith("Follow ") and not p_clean.startswith("Read more") and not "click here" in p_clean.lower() and not "twitter" in p_clean.lower(): + paragraphs.append(p_clean) + + # Keep top 8 substantial paragraphs + full_text = "\n\n".join(paragraphs[:8]) + return full_text, img_url + except Exception as e: + print(f"Scraping error for {url}: {e}") + return "", "" + while True: payload = {} for category, url in feeds.items(): @@ -885,19 +918,16 @@ if __name__ == "__main__": stories = [] channel = root.find("channel") if channel is not None: - # Fetch up to 10 stories per category + # Scrape top 5 rich stories per category to keep it fast for idx, item in enumerate(channel.findall("item")): - if idx >= 10: + if idx >= 5: break title = item.find("title") - desc = item.find("description") + link = item.find("link") pub_date = item.find("pubDate") t_str = title.text.strip() if title is not None and title.text else "" - d_str = desc.text.strip() if desc is not None and desc.text else "" - - # Clean HTML tags from description if any - d_str = re.sub('<[^<]+?>', '', d_str) + l_str = link.text.strip() if link is not None and link.text else "" p_str = "" if pub_date is not None and pub_date.text: @@ -905,14 +935,20 @@ if __name__ == "__main__": p_str = " ".join(pub_date.text.split()[:4]) except Exception: p_str = pub_date.text - + + # Perform background scraping for full text and image + full_story, img_url = "", "" + if l_str: + full_story, img_url = scrape_article(l_str) + stories.append({ "title": t_str, - "summary": d_str, + "summary": full_story if full_story else "Click to read the story details.", + "image": img_url, "date": p_str }) payload[category] = stories - print(f"Successfully cached {len(stories)} stories for category: {category}") + print(f"Successfully cached and scraped {len(stories)} stories for category: {category}") except Exception as e: print(f"Failed to fetch RSS category {category}: {e}") @@ -920,7 +956,7 @@ if __name__ == "__main__": try: cache_file.parent.mkdir(parents=True, exist_ok=True) cache_file.write_text(json.dumps(payload, indent=2)) - print("Updated local news-data.json cache successfully") + print("Updated local news-data.json cache with full stories successfully") except Exception as e: print(f"Failed to write news cache file: {e}") diff --git a/news.html b/news.html index c119b83..1865b49 100644 --- a/news.html +++ b/news.html @@ -246,6 +246,12 @@ const card = document.createElement('div'); card.className = 'story-card'; + // Wrap each scraped paragraph in a comfortable

block for readability + const formattedSummary = story.summary + .split('\n\n') + .map(para => `

${para}

`) + .join(''); + card.innerHTML = `
${story.title}
@@ -253,7 +259,8 @@
-
${story.summary}
+ ${story.image ? `` : ''} +
${formattedSummary}