X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=generic_news_rss_renderer.py;h=f1261cb59e171cfa2a8348487839cb7edc2dd124;hb=5ea88ab72e175e2d4f57ae8645ca6f825549a7a9;hp=1ffe024a7e9d3d798e6432804badcb2c430a18f4;hpb=c6d31748857aeb8f361dadf93bd00e04d60bfb70;p=kiosk.git
diff --git a/generic_news_rss_renderer.py b/generic_news_rss_renderer.py
index 1ffe024..f1261cb 100644
--- a/generic_news_rss_renderer.py
+++ b/generic_news_rss_renderer.py
@@ -17,7 +17,7 @@ import renderer
import page_builder
-logger = logging.getLogger(__file__)
+logger = logging.getLogger(__name__)
class generic_news_rss_renderer(renderer.abstaining_renderer):
@@ -55,7 +55,7 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
pass
def should_profanity_filter(self) -> bool:
- return False
+ return True
def find_title(self, item: ET.Element) -> Optional[str]:
return item.findtext("title")
@@ -66,11 +66,7 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
def find_description(self, item: ET.Element) -> Optional[str]:
return item.findtext("description")
- def munge_description(
- self,
- description: str,
- item: ET.Element
- ) -> str:
+ def munge_description(self, description: str, item: ET.Element) -> str:
description = re.sub("<[^>]+>", "", description)
return description
@@ -103,10 +99,12 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
def do_details(self) -> bool:
return True
- def is_item_older_than_n_days(self, item: ET.Element, n: int) -> bool:
+ def is_item_older_than_n_days(
+ self, item: ET.Element, n: int, default: bool = False
+ ) -> bool:
pubdate = self.find_pubdate(item)
if pubdate is None:
- return False
+ return default
pubdatetime = parse(pubdate)
tzinfo = pubdatetime.tzinfo
now = datetime.datetime.now(tzinfo)
@@ -133,7 +131,7 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
headlines.set_title("%s" % self.page_title)
subset = self.news.subset(4)
if subset is None:
- logger.warning('Not enough messages to select from in shuffle_news?!')
+ logger.warning("Not enough messages to select from in shuffle_news?!")
return False
for msg in subset:
headlines.add_item(msg)
@@ -187,7 +185,7 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
details.set_title(self.page_title)
subset = self.details.subset(1)
if subset is None:
- logger.warning('Not enough details to choose from in do_details')
+ logger.warning("Not enough details to choose from in do_details")
logger.debug("Not enough details to choose from.")
return False
for msg in subset:
@@ -203,18 +201,19 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
count = 0
self.news.clear()
self.details.clear()
- self.conn: Optional[Union[http.client.HTTPConnection,
- http.client.HTTPSConnection]] = None
+ self.conn: Optional[
+ Union[http.client.HTTPConnection, http.client.HTTPSConnection]
+ ] = None
for uri in self.feed_uris:
url = None
if self.should_use_https():
- url = f'https://{self.feed_site}{uri}'
- logger.info(f'Fetching: {url}')
+ url = f"https://{self.feed_site}{uri}"
+ logger.info(f"Fetching: {url}")
self.conn = http.client.HTTPSConnection(self.feed_site, timeout=10)
else:
- url = f'http://{self.feed_site}{uri}'
- logger.info(f'Fetching: {url}')
+ url = f"http://{self.feed_site}{uri}"
+ logger.info(f"Fetching: {url}")
self.conn = http.client.HTTPConnection(self.feed_site, timeout=10)
assert self.conn is not None
assert url is not None
@@ -229,21 +228,22 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
)
try:
response = self.conn.getresponse()
- except Exception as e:
- logger.exception(e)
- logger.error(
+ except Exception:
+ logger.exception(
f"Exception in generic RSS renderer HTTP connection fetching {url}; giving up."
)
return False
if response.status != 200:
logger.error(
- f'Unexpected status {response.status} while fetching {url}; giving up.'
+ f"Unexpected status {response.status} while fetching {url}: {response.reason}; giving up."
)
+ print(dir(response))
+ print(response.headers)
return False
raw = response.read()
- logger.info(f'Status 200: got {len(raw)} bytes back from {url}')
+ logger.info(f"Status 200: got {len(raw)} bytes back from {url}")
rss = ET.fromstring(raw)
channel = rss[0]
title_filter = set()
@@ -253,9 +253,9 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
if title is not None:
title = self.munge_title(title, item)
else:
- logger.info('Skipping RSS feed item with no title.')
+ logger.info("Skipping RSS feed item with no title.")
continue
- logger.debug(f'Considering RSS item {title}...')
+ logger.debug(f"Considering RSS item {title}...")
if description is not None:
description = self.munge_description(description, item)
else:
@@ -266,21 +266,21 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
link = item.findtext("link")
if link is not None:
link = self.munge_link(link)
- if not self.item_is_interesting_for_headlines(
- title, description, item
- ):
- logger.info(f'Skipping {title} because it\'s not interesting.')
+ if not self.item_is_interesting_for_headlines(title, description, item):
+ logger.info(f"Skipping {title} because it's not interesting.")
continue
if self.should_profanity_filter() and (
self.filter.contains_bad_word(title)
or self.filter.contains_bad_word(description)
):
- logger.info(f'Skipping {title} because it contains profanity.')
+ logger.info(f"Skipping {title} because it contains profanity.")
continue
if title in title_filter:
- logger.info(f'Skipping {title} because we already saw an item with the same title.')
+ logger.info(
+ f"Skipping {title} because we already saw an item with the same title."
+ )
continue
title_filter.add(title)
@@ -298,14 +298,16 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
pubdate = self.find_pubdate(item)
if pubdate is not None:
- logger.debug(f'Raw pubdate={pubdate}')
+ logger.debug(f"Raw pubdate={pubdate}")
pubdate = self.munge_pubdate(pubdate)
ts = parse(pubdate)
- logger.debug(f'Translated pubdate into: {ts}')
+ logger.debug(f"Translated pubdate into: {ts}")
blurb += f' {ts.strftime("%b %d")}'
if self.item_is_interesting_for_article(title, description, item):
- logger.info(f'Item {title} is also interesting as an article details page; creating...')
+ logger.info(
+ f"Item {title} is also interesting as an article details page; creating..."
+ )
longblurb = blurb
longblurb += "
"
longblurb += description
@@ -313,9 +315,11 @@ class generic_news_rss_renderer(renderer.abstaining_renderer):
longblurb = longblurb.replace("font-size:34pt", "font-size:44pt")
self.details.add(longblurb)
else:
- logger.info(f'Item {title} isn\'t interesting for article details page; skipped.')
+ logger.info(
+ f"Item {title} isn't interesting for article details page; skipped."
+ )
blurb += ""
self.news.add(blurb)
count += 1
- logger.debug(f'Added {count} items so far...')
+ logger.debug(f"Added {count} items so far...")
return count > 0