X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=generic_news_rss_renderer.py;h=71cf7ed2d0ceee7191f69d818684e5b65dfc17ab;hb=e4dca16bbd329afdb587e8488767d88e17777254;hp=4db466957456095148d1793a772e0069b02ae32b;hpb=78b904e30bc0f9a05a96da1b597ad11795afbd46;p=kiosk.git diff --git a/generic_news_rss_renderer.py b/generic_news_rss_renderer.py index 4db4669..71cf7ed 100644 --- a/generic_news_rss_renderer.py +++ b/generic_news_rss_renderer.py @@ -1,20 +1,31 @@ +#!/usr/bin/env python3 + +from abc import abstractmethod import datetime from dateutil.parser import parse +import http.client +import random +import re +from typing import Dict, List, Optional, Union +import xml.etree.ElementTree as ET + import file_writer import grab_bag import renderer -import http.client import page_builder import profanity_filter -import random -import re -import xml.etree.ElementTree as ET + class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): - def __init__(self, name_to_timeout_dict, feed_site, feed_uris, page_title): - super(generic_news_rss_renderer, self).__init__(name_to_timeout_dict, - False) - self.debug = 1 + def __init__( + self, + name_to_timeout_dict: Dict[str, int], + feed_site: str, + feed_uris: List[str], + page_title: str, + ): + super(generic_news_rss_renderer, self).__init__(name_to_timeout_dict, False) + self.debug = True self.feed_site = feed_site self.feed_uris = feed_uris self.page_title = page_title @@ -22,72 +33,91 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): self.details = grab_bag.grab_bag() self.filter = profanity_filter.profanity_filter() - def debug_prefix(self): + @abstractmethod + def debug_prefix(self) -> str: pass - def get_headlines_page_prefix(self): + @abstractmethod + def get_headlines_page_prefix(self) -> str: pass - def get_details_page_prefix(self): + @abstractmethod + def get_details_page_prefix(self) -> str: pass - def should_use_https(self): + def get_headlines_page_priority(self) -> str: + return "4" + + def get_details_page_priority(self) -> str: + return "6" + + @abstractmethod + def should_use_https(self) -> bool: pass - def should_profanity_filter(self): + def should_profanity_filter(self) -> bool: return False - def find_title(self, item): - return item.findtext('title') + def find_title(self, item: ET.Element) -> Optional[str]: + return item.findtext("title") - def munge_title(self, title): + def munge_title(self, title: str) -> str: return title - def find_description(self, item): - return item.findtext('description') + def find_description(self, item: ET.Element) -> Optional[str]: + return item.findtext("description") - def munge_description(self, description): - description = re.sub('<[^>]+>', '', description) + def munge_description(self, description: str) -> str: + description = re.sub("<[^>]+>", "", description) return description - def find_link(self, item): - return item.findtext('link') + def find_link(self, item: ET.Element) -> Optional[str]: + return item.findtext("link") - def munge_link(self, link): + def munge_link(self, link: str) -> str: return link - def find_image(self, item): - return item.findtext('image') + def find_image(self, item: ET.Element) -> Optional[str]: + return item.findtext("image") - def munge_image(self, image): + def munge_image(self, image: str) -> str: return image - def item_is_interesting_for_headlines(self, title, description, item): + def find_pubdate(self, item: ET.Element) -> Optional[str]: + return item.findtext("pubDate") + + def munge_pubdate(self, pubdate: str) -> str: + return pubdate + + def item_is_interesting_for_headlines( + self, title: str, description: str, item: ET.Element + ) -> bool: return True - def is_item_older_than_n_days(self, item, n): - pubdate = item.findtext('pubDate') - if pubdate is not None: - pubdate = parse(pubdate) - tzinfo = pubdate.tzinfo - now = datetime.datetime.now(tzinfo) - delta = (now - pubdate).total_seconds() / (60 * 60 * 24) - if (delta > n): - return True - return False + def is_item_older_than_n_days(self, item: ET.Element, n: int) -> bool: + pubdate = self.find_pubdate(item) + if pubdate is None: + return False + pubdatetime = parse(pubdate) + tzinfo = pubdatetime.tzinfo + now = datetime.datetime.now(tzinfo) + delta = (now - pubdatetime).total_seconds() / (60 * 60 * 24) + return delta > n - def item_is_interesting_for_article(self, title, description, item): + def item_is_interesting_for_article( + self, title: str, description: str, item: ET.Element + ) -> bool: return True - def periodic_render(self, key): + def periodic_render(self, key: str) -> bool: if key == "Fetch News": return self.fetch_news() elif key == "Shuffle News": return self.shuffle_news() else: - raise error('Unexpected operation') + raise Exception - def shuffle_news(self): + def shuffle_news(self) -> bool: headlines = page_builder.page_builder() headlines.set_layout(page_builder.page_builder.LAYOUT_FOUR_ITEMS) headlines.set_title("%s" % self.page_title) @@ -97,86 +127,102 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): return False for msg in subset: headlines.add_item(msg) - headlines.set_custom_html(""" + headlines.set_custom_html( + """ """) - f = file_writer.file_writer('%s_4_none.html' % ( - self.get_headlines_page_prefix())) - headlines.render_html(f) - f.close() +""" + ) + _ = f"{self.get_headlines_page_prefix()}_{self.get_headlines_page_priority()}_25900.html" + with file_writer.file_writer(_) as f: + headlines.render_html(f) details = page_builder.page_builder() details.set_layout(page_builder.page_builder.LAYOUT_ONE_ITEM) - details.set_custom_html(""" + details.set_custom_html( + """ """) - details.set_title("%s" % self.page_title) +""" + ) + details.set_title(f"{self.page_title}") subset = self.details.subset(1) if subset is None: - self.debug_print("Not enough details to choose from."); + self.debug_print("Not enough details to choose from.") return False for msg in subset: blurb = msg - blurb += u'' + blurb += "" details.add_item(blurb) - g = file_writer.file_writer('%s_6_none.html' % ( - self.get_details_page_prefix())) - details.render_html(g) - g.close() + _ = f"{self.get_details_page_prefix()}_{self.get_details_page_priority()}_86400.html" + with file_writer.file_writer(_) as g: + details.render_html(g) return True - def fetch_news(self): + def fetch_news(self) -> bool: count = 0 self.news.clear() self.details.clear() + self.conn: Optional[Union[http.client.HTTPConnection, + http.client.HTTPSConnection]] = None for uri in self.feed_uris: if self.should_use_https(): self.debug_print("Fetching: https://%s%s" % (self.feed_site, uri)) - self.conn = http.client.HTTPSConnection(self.feed_site) + self.conn = http.client.HTTPSConnection(self.feed_site, timeout=20) else: self.debug_print("Fetching: http://%s%s" % (self.feed_site, uri)) - self.conn = http.client.HTTPConnection(self.feed_site) + self.conn = http.client.HTTPConnection(self.feed_site, timeout=20) + assert(self.conn is not None) self.conn.request( "GET", uri, None, - {"Accept-Charset": "utf-8"}) - response = self.conn.getresponse() + { + "Accept": "*/*", + "Cache-control": "max-age=59", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + }, + ) + try: + response = self.conn.getresponse() + except: + print("Exception in generic RSS renderer HTTP connection") + return False + if response.status != 200: - print(("%s: RSS fetch_news error, response: %d" % (self.page_title, - response.status))) - self.debug_print(response.read()) + print( + f"{self.page_title}: RSS fetch_news error, response: {response.status}" + ) + self.debug_print(str(response.read())) return False rss = ET.fromstring(response.read()) @@ -185,57 +231,57 @@ a:active { title = self.find_title(item) if title is not None: title = self.munge_title(title) - description = item.findtext('description') + description = item.findtext("description") if description is not None: description = self.munge_description(description) + else: + description = "" image = self.find_image(item) if image is not None: image = self.munge_image(image) - link = item.findtext('link') + link = item.findtext("link") if link is not None: link = self.munge_link(link) - if (title is None or - not self.item_is_interesting_for_headlines(title, - description, - item)): - self.debug_print('Item "%s" is not interesting' % title) + if title is None or not self.item_is_interesting_for_headlines( + title, description, item + ): + self.debug_print(f'Item "{title}" is not interesting') continue - if (self.should_profanity_filter() and - (self.filter.contains_bad_words(title) or - self.filter.contains_bad_words(description))): - self.debug_print('Found bad words in item "%s"' % title) + if self.should_profanity_filter() and ( + self.filter.contains_bad_words(title) + or self.filter.contains_bad_words(description) + ): + self.debug_print(f'Found bad words in item "{title}"') continue - #print u"Title: %s\nDescription: %s\nLink: %s\nImage: %s\n" % ( - # title, description, link, image) - - blurb = u"""
""" if image is not None: - blurb += u'' + blurb += f'%s' % title + blurb += f"

{title}" else: - blurb += u'

%s' % (link, title) + blurb += f'

{title}' - if (description is not None and - self.item_is_interesting_for_article(title, - description, - item)): + pubdate = self.find_pubdate(item) + if pubdate is not None: + pubdate = self.munge_pubdate(pubdate) + ts = parse(pubdate) + blurb += f' {ts.strftime("%b %d")}' + + if self.item_is_interesting_for_article(title, description, item): longblurb = blurb - longblurb += u"
" + longblurb += "
" longblurb += description - longblurb += u"

" - longblurb = longblurb.replace("font-size:34pt", - "font-size:44pt") + longblurb += "" + longblurb = longblurb.replace("font-size:34pt", "font-size:44pt") self.details.add(longblurb) - - blurb += u"" + blurb += "" self.news.add(blurb) count += 1 return count > 0