feat: upgrade local newspaper to full-text and images

browser.py

@@ -864,7 +864,7 @@ if __name__ == "__main__":
             time.sleep(900)
 
     def update_news_cache():
-        """Fetch top RSS news feeds (BBC, NPR) in background and cache as local JSON to bypass CORS/Ads."""
+        """Fetch top RSS news feeds (BBC, NPR), scrape full article text & images in background, and cache locally."""
         import xml.etree.ElementTree as ET
         import re
         cache_file = _HOME / "kiosk-home" / "news-data.json"
@@ -874,6 +874,39 @@ if __name__ == "__main__":
             "National News": "https://feeds.npr.org/1001/rss.xml"
         }
         
+        def scrape_article(url):
+            try:
+                req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
+                html = urllib.request.urlopen(req, timeout=10).read().decode('utf-8', errors='ignore')
+                
+                # 1. Extract og:image
+                img_url = ""
+                og_img_match = re.search(r'<meta[^>]*property=["\']og:image["\'][^>]*content=["\']([^"\']+)["\']', html)
+                if not og_img_match:
+                    og_img_match = re.search(r'<meta[^>]*content=["\']([^"\']+)["\'][^>]*property=["\']og:image["\']', html)
+                if og_img_match:
+                    img_url = og_img_match.group(1)
+                
+                # 2. Extract main paragraphs
+                paragraphs = []
+                p_matches = re.findall(r'<p[^>]*>(.*?)</p>', html, re.DOTALL)
+                for p in p_matches:
+                    p_clean = re.sub('<[^<]+?>', '', p)
+                    p_clean = p_clean.strip()
+                    # Decode basic HTML entities
+                    p_clean = p_clean.replace("&nbsp;", " ").replace("&amp;", "&").replace("&quot;", '"').replace("&apos;", "'").replace("&#39;", "'").replace("&ndash;", "-").replace("&mdash;", "-")
+                    
+                    # Filter out short noise paragraphs (ads, nav links)
+                    if len(p_clean) > 85 and not p_clean.startswith("Follow ") and not p_clean.startswith("Read more") and not "click here" in p_clean.lower() and not "twitter" in p_clean.lower():
+                        paragraphs.append(p_clean)
+                
+                # Keep top 8 substantial paragraphs
+                full_text = "\n\n".join(paragraphs[:8])
+                return full_text, img_url
+            except Exception as e:
+                print(f"Scraping error for {url}: {e}")
+                return "", ""
+
         while True:
             payload = {}
             for category, url in feeds.items():
@@ -885,19 +918,16 @@ if __name__ == "__main__":
                     stories = []
                     channel = root.find("channel")
                     if channel is not None:
-                        # Fetch up to 10 stories per category
+                        # Scrape top 5 rich stories per category to keep it fast
                         for idx, item in enumerate(channel.findall("item")):
-                            if idx >= 10:
+                            if idx >= 5:
                                 break
                             title = item.find("title")
-                            desc = item.find("description")
+                            link = item.find("link")
                             pub_date = item.find("pubDate")
                             
                             t_str = title.text.strip() if title is not None and title.text else ""
-                            d_str = desc.text.strip() if desc is not None and desc.text else ""
+                            l_str = link.text.strip() if link is not None and link.text else ""
-                            
-                            # Clean HTML tags from description if any
-                            d_str = re.sub('<[^<]+?>', '', d_str)
                             
                             p_str = ""
                             if pub_date is not None and pub_date.text:
@@ -905,14 +935,20 @@ if __name__ == "__main__":
                                     p_str = " ".join(pub_date.text.split()[:4])
                                 except Exception:
                                     p_str = pub_date.text
-                                    
+                            
+                            # Perform background scraping for full text and image
+                            full_story, img_url = "", ""
+                            if l_str:
+                                full_story, img_url = scrape_article(l_str)
+                            
                             stories.append({
                                 "title": t_str,
-                                "summary": d_str,
+                                "summary": full_story if full_story else "Click to read the story details.",
+                                "image": img_url,
                                 "date": p_str
                             })
                     payload[category] = stories
-                    print(f"Successfully cached {len(stories)} stories for category: {category}")
+                    print(f"Successfully cached and scraped {len(stories)} stories for category: {category}")
                 except Exception as e:
                     print(f"Failed to fetch RSS category {category}: {e}")
             
@@ -920,7 +956,7 @@ if __name__ == "__main__":
                 try:
                     cache_file.parent.mkdir(parents=True, exist_ok=True)
                     cache_file.write_text(json.dumps(payload, indent=2))
-                    print("Updated local news-data.json cache successfully")
+                    print("Updated local news-data.json cache with full stories successfully")
                 except Exception as e:
                     print(f"Failed to write news cache file: {e}")
             

news.html

@@ -246,6 +246,12 @@
         const card = document.createElement('div');
         card.className = 'story-card';
 
+        // Wrap each scraped paragraph in a comfortable <p> block for readability
+        const formattedSummary = story.summary
+          .split('\n\n')
+          .map(para => `<p style="margin-bottom:18px;">${para}</p>`)
+          .join('');
+
         card.innerHTML = `
           <div class="story-header" onclick="toggleStory(this)">
             <div class="story-headline">${story.title}</div>
@@ -253,7 +259,8 @@
           </div>
           <div class="story-body">
             <div class="story-content">
-              <div class="story-summary">${story.summary}</div>
+              ${story.image ? `<img src="${story.image}" style="width:100%; max-height:450px; object-fit:cover; border-radius:16px; margin-bottom:24px; box-shadow: 0 4px 15px rgba(0,0,0,0.35);" onerror="this.style.display='none'">` : ''}
+              <div class="story-summary">${formattedSummary}</div>
               <div class="story-meta">⌚ Published: ${story.date || 'Recent'}</div>
             </div>
           </div>