From 6b8d4eeb7153617221f822a243a117f0bcab07bf Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Sat, 13 Nov 2021 07:03:58 -0800 Subject: [PATCH] Somewhat large overhaul to move the kiosk towards using normal python logging. --- bellevue_city_calendar_renderer.py | 10 +- bellevue_reporter_rss_renderer.py | 26 +- chooser.py | 42 +-- cnn_rss_renderer.py | 6 +- constants.py | 4 +- gcal_renderer.py | 32 +-- generic_news_rss_renderer.py | 75 +++--- gkeep_renderer.py | 31 ++- google_news_rss_renderer.py | 14 +- grab_bag.py | 5 +- health_renderer.py | 17 +- kiosk.py | 403 +++++++++++++++++------------ listen.py | 38 ++- local_photos_mirror_renderer.py | 4 +- mynorthwest_rss_renderer.py | 2 +- myq_renderer.py | 6 +- reddit_renderer.py | 62 +++-- renderer.py | 53 ++-- renderer_catalog.py | 7 - seattletimes_rss_renderer.py | 34 ++- stock_renderer.py | 16 +- stranger_renderer.py | 30 ++- twitter_renderer.py | 7 +- urbanist_renderer.py | 2 +- weather_renderer.py | 9 +- wsj_rss_renderer.py | 2 +- 26 files changed, 518 insertions(+), 419 deletions(-) diff --git a/bellevue_city_calendar_renderer.py b/bellevue_city_calendar_renderer.py index 4587569..c55480b 100644 --- a/bellevue_city_calendar_renderer.py +++ b/bellevue_city_calendar_renderer.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import datetime +import logging import re from typing import Dict, List, Optional import xml @@ -10,6 +11,9 @@ from dateutil.parser import parse import generic_news_rss_renderer as gnrss +logger = logging.getLogger(__file__) + + class bellevue_city_calendar_renderer(gnrss.generic_news_rss_renderer): """Read the Bellevue City Calendar feed.""" @@ -20,13 +24,9 @@ class bellevue_city_calendar_renderer(gnrss.generic_news_rss_renderer): feed_uris: List[str], page_title: str, ): - super(bellevue_city_calendar_renderer, self).__init__( + super().__init__( name_to_timeout_dict, feed_site, feed_uris, page_title ) - self.debug = True - - def debug_prefix(self) -> str: - return f"bellevue_calendar({self.page_title})" def get_headlines_page_prefix(self) -> str: return "bellevue-calendar" diff --git a/bellevue_reporter_rss_renderer.py b/bellevue_reporter_rss_renderer.py index b8fd27b..4e1ff62 100644 --- a/bellevue_reporter_rss_renderer.py +++ b/bellevue_reporter_rss_renderer.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import logging import re from typing import List, Dict import xml @@ -8,6 +9,9 @@ import xml.etree.ElementTree as ET import generic_news_rss_renderer as gnrss +logger = logging.getLogger(__file__) + + class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): """Read the Bellevue Reporter's RSS feed.""" @@ -18,13 +22,9 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): feed_uris: List[str], page_title: str, ): - super(bellevue_reporter_rss_renderer, self).__init__( + super().__init__( name_to_timeout_dict, feed_site, feed_uris, page_title ) - self.debug = True - - def debug_prefix(self) -> str: - return f"bellevue_reporter({self.page_title})" def get_headlines_page_prefix(self) -> str: return "bellevue-reporter" @@ -75,16 +75,16 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): ) -> bool: unfiltered_description = item.findtext("description") if self.is_item_older_than_n_days(item, 10): - self.debug_print(f'{title}: is too old!') + logger.info(f'{title}: is too old!') return False if bellevue_reporter_rss_renderer.looks_like_spam(title, unfiltered_description): - self.debug_print(f'{title}: looks like spam') + logger.debug(f'{title}: looks like spam') return False if bellevue_reporter_rss_renderer.looks_like_football(title, description): - self.debug_print(f'{title}: looks like it\'s about football.') + logger.debug(f'{title}: looks like it\'s about football.') return False if bellevue_reporter_rss_renderer.looks_like_review(title, description): - self.debug_print(f'{title}: looks like a review.') + logger.debug(f'{title}: looks like a review.') return False return True @@ -93,16 +93,16 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): ) -> bool: unfiltered_description = item.findtext("description") if self.is_item_older_than_n_days(item, 10): - self.debug_print(f'{title}: is too old!') + logger.debug(f'{title}: is too old!') return False if bellevue_reporter_rss_renderer.looks_like_spam(title, unfiltered_description): - self.debug_print(f'{title}: looks like spam') + logger.debug(f'{title}: looks like spam') return False if bellevue_reporter_rss_renderer.looks_like_football(title, description): - self.debug_print(f'{title}: looks like it\'s about football.') + logger.debug(f'{title}: looks like it\'s about football.') return False if bellevue_reporter_rss_renderer.looks_like_review(title, description): - self.debug_print(f'{title}: looks like a review.') + logger.debug(f'{title}: looks like a review.') return False return True diff --git a/chooser.py b/chooser.py index beffdb2..3514c97 100644 --- a/chooser.py +++ b/chooser.py @@ -1,22 +1,28 @@ #!/usr/bin/env python3 from abc import ABC, abstractmethod -import datetime -import glob +import logging import os import random import re -import sys import time from typing import Any, Callable, List, Optional, Set, Tuple +import datetime_utils + import constants import trigger +logger = logging.getLogger(__file__) + + class chooser(ABC): """Base class of a thing that chooses pages""" + def __init__(self): + pass + def get_page_list(self) -> List[str]: now = time.time() valid_filename = re.compile("([^_]+)_(\d+)_([^\.]+)\.html") @@ -29,7 +35,6 @@ class chooser(ABC): for page in pages: result = re.match(valid_filename, page) if result is not None: - print(f'chooser: candidate page: "{page}"') if result.group(3) != "none": freshness_requirement = int(result.group(3)) last_modified = int( @@ -37,8 +42,13 @@ class chooser(ABC): ) age = now - last_modified if age > freshness_requirement: - print(f'chooser: "{page}" is too old.') + logger.warning( + f'chooser: "{page}" is too old.' + ) continue + logger.info( + f'chooser: candidate page: "{page}"' + ) filenames.append(page) return filenames @@ -51,7 +61,8 @@ class weighted_random_chooser(chooser): """Chooser that does it via weighted RNG.""" def __init__(self, filter_list: Optional[List[Callable[[str], bool]]]) -> None: - self.last_choice = "" + super().__init__() + self.last_choice = None self.valid_filename = re.compile("([^_]+)_(\d+)_([^\.]+)\.html") self.pages: Optional[List[str]] = None self.count = 0 @@ -61,13 +72,14 @@ class weighted_random_chooser(chooser): self.filter_list.append(self.dont_choose_page_twice_in_a_row_filter) def dont_choose_page_twice_in_a_row_filter(self, choice: str) -> bool: - if choice == self.last_choice: + if self.last_choice is not None and choice == self.last_choice: return False self.last_choice = choice return True def choose_next_page(self) -> Any: if self.pages is None or self.count % 100 == 0: + logger.info('chooser: refreshing the candidate pages list.') self.pages = self.get_page_list() total_weight = 0 @@ -90,11 +102,10 @@ class weighted_random_chooser(chooser): break choice = self.pages[x] - # Allow filter list to suppress pages. + # Allow filters list to suppress pages. choice_is_filtered = False for f in self.filter_list: if not f(choice): - print(f"chooser: {choice} filtered by {f.__name__}") choice_is_filtered = True break if choice_is_filtered: @@ -113,7 +124,7 @@ class weighted_random_chooser_with_triggers(weighted_random_chooser): trigger_list: Optional[List[trigger.trigger]], filter_list: List[Callable[[str], bool]], ) -> None: - weighted_random_chooser.__init__(self, filter_list) + super().__init__(filter_list) self.trigger_list: List[trigger.trigger] = [] if trigger_list is not None: self.trigger_list.extend(trigger_list) @@ -126,19 +137,20 @@ class weighted_random_chooser_with_triggers(weighted_random_chooser): if x is not None and len(x) > 0: for y in x: self.page_queue.add(y) + logger.info(f'chooser: noticed active trigger {y}') triggered = True return triggered def choose_next_page(self) -> Tuple[str, bool]: if self.pages is None or self.count % 100 == 0: + logger.info('chooser: refreshing the candidates page list') self.pages = self.get_page_list() triggered = self.check_for_triggers() # First try to satisfy from the page queue. - now = datetime.datetime.now() if len(self.page_queue) > 0: - print("chooser: Pulling page from queue...") + logger.info('chooser: page queue has entries; pulling choice from there.') page = None priority = None for t in self.page_queue: @@ -151,14 +163,14 @@ class weighted_random_chooser_with_triggers(weighted_random_chooser): return (page, triggered) # Always show the clock in the middle of the night. - elif now.hour < 6: + now = datetime_utils.now_pacific() + if now.hour < 6: for page in self.pages: if "clock" in page: return (page, False) # Fall back on weighted random choice. - else: - return (weighted_random_chooser.choose_next_page(self), False) + return (weighted_random_chooser.choose_next_page(self), False) # Test diff --git a/cnn_rss_renderer.py b/cnn_rss_renderer.py index 3e15c98..a4c8945 100644 --- a/cnn_rss_renderer.py +++ b/cnn_rss_renderer.py @@ -14,13 +14,9 @@ class cnn_rss_renderer(generic_news_rss_renderer.generic_news_rss_renderer): feed_uris: List[str], page_title: str, ): - super(cnn_rss_renderer, self).__init__( + super().__init__( name_to_timeout_dict, feed_site, feed_uris, page_title ) - self.debug = True - - def debug_prefix(self) -> str: - return f"cnn({self.page_title})" def get_headlines_page_prefix(self) -> str: return f"cnn-{self.page_title}" diff --git a/constants.py b/constants.py index a4f8e87..ce04f7e 100644 --- a/constants.py +++ b/constants.py @@ -5,13 +5,15 @@ pages_dir = '/var/www/html/kiosk' root_url = f'http://{hostname}/kiosk' refresh_period_sec = 22.0 +emergency_refresh_period_sec = 45.0 refresh_period_night_sec = 600.0 render_period_sec = 30.0 +check_threads_period_sec = 60.0 seconds_per_minute = 60 seconds_per_hour = seconds_per_minute * 60 seconds_per_day = seconds_per_hour * 24 myq_pagename = "myq_4_300.html" -internal_stats_pagename = 'internal-stats_1_1000.html' +render_stats_pagename = 'internal/render-stats_1_1000.html' gcal_imminent_pagename = "hidden/gcal-imminent_0_none.html" diff --git a/gcal_renderer.py b/gcal_renderer.py index c43a144..19b818d 100644 --- a/gcal_renderer.py +++ b/gcal_renderer.py @@ -5,24 +5,24 @@ contents of several Google calendars.""" import datetime import functools -import os +import logging import time from typing import Any, Dict, List, Optional, Tuple from dateutil.parser import parse -import gdata # type: ignore import gdata_oauth -from oauth2client.client import AccessTokenRefreshError # type: ignore import pytz import constants import file_writer import globals import renderer -import kiosk_secrets as secrets -class gcal_renderer(renderer.debuggable_abstaining_renderer): +logger = logging.getLogger(__file__) + + +class gcal_renderer(renderer.abstaining_renderer): """A renderer to fetch upcoming events from www.google.com/calendar""" calendar_whitelist = frozenset( @@ -123,7 +123,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): def __init__( self, name_to_timeout_dict: Dict[str, int], oauth: gdata_oauth.OAuth ) -> None: - super(gcal_renderer, self).__init__(name_to_timeout_dict, True) + super().__init__(name_to_timeout_dict) self.oauth = oauth self.client = self.oauth.calendar_service() self.sortable_events: List[gcal_renderer.comparable_event] = [] @@ -133,7 +133,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): return "gcal" def periodic_render(self, key: str) -> bool: - self.debug_print('called for "%s"' % key) + logger.debug('called for "%s"' % key) if key == "Render Upcoming Events": return self.render_upcoming_events() elif key == "Look For Triggered Events": @@ -147,8 +147,8 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): _time_max = now + datetime.timedelta(days=95) time_min = datetime.datetime.strftime(_time_min, "%Y-%m-%dT%H:%M:%SZ") time_max = datetime.datetime.strftime(_time_max, "%Y-%m-%dT%H:%M:%SZ") - self.debug_print(f"time_min is {time_min}") - self.debug_print(f"time_max is {time_max}") + logger.debug(f"time_min is {time_min}") + logger.debug(f"time_max is {time_max}") return (time_min, time_max) @staticmethod @@ -182,7 +182,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): ) for calendar in calendar_list["items"]: if calendar["summary"] in gcal_renderer.calendar_whitelist: - self.debug_print( + logger.debug( f"{calendar['summary']} is an interesting calendar..." ) events = ( @@ -200,11 +200,11 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): summary = event["summary"] start = gcal_renderer.parse_date(event["start"]) end = gcal_renderer.parse_date(event["end"]) - self.debug_print( + logger.debug( f" ... event '{summary}' ({event['start']} ({start}) to {event['end']} ({end})" ) if start is not None and end is not None: - self.debug_print(f' ... adding {summary} to sortable_events') + logger.debug(f' ... adding {summary} to sortable_events') sortable_events.append( gcal_renderer.comparable_event( start, end, summary, calendar["summary"] @@ -215,7 +215,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): or "Holidays" in calendar["summary"] or "Countdown" in summary ): - self.debug_print(f" ... adding {summary} to countdown_events") + logger.debug(f" ... adding {summary} to countdown_events") countdown_events.append( gcal_renderer.comparable_event( start, end, summary, calendar["summary"] @@ -246,7 +246,7 @@ f""" ) upcoming_sortable_events = self.sortable_events[:12] for n, event in enumerate(upcoming_sortable_events): - self.debug_print(f'{n}/12: {event.friendly_name()} / {event.calendar}') + logger.debug(f'{n}/12: {event.friendly_name()} / {event.calendar}') if n % 2 == 0: color = "#c6b0b0" else: @@ -297,7 +297,7 @@ f""" ) timestamps[identifier] = time.mktime(eventstamp.timetuple()) count += 1 - self.debug_print( + logger.debug( "countdown to %s is %dd %dh %dm" % (name, days[0], hours[0], minutes[0]) ) @@ -341,7 +341,7 @@ var fn = setInterval(function() { """ ) return True - except (gdata.service.RequestError, AccessTokenRefreshError): + except Exception as e: print("********* TRYING TO REFRESH GCAL CLIENT *********") # self.oauth.refresh_token() # self.client = self.oauth.calendar_service() diff --git a/generic_news_rss_renderer.py b/generic_news_rss_renderer.py index 149f8ac..61be6ff 100644 --- a/generic_news_rss_renderer.py +++ b/generic_news_rss_renderer.py @@ -4,10 +4,8 @@ from abc import abstractmethod import datetime from dateutil.parser import parse import http.client -import random +import logging import re -import sys -import traceback from typing import Dict, List, Optional, Union import xml.etree.ElementTree as ET @@ -18,7 +16,10 @@ import page_builder import profanity_filter -class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): +logger = logging.getLogger(__file__) + + +class generic_news_rss_renderer(renderer.abstaining_renderer): def __init__( self, name_to_timeout_dict: Dict[str, int], @@ -26,8 +27,7 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): feed_uris: List[str], page_title: str, ): - super(generic_news_rss_renderer, self).__init__(name_to_timeout_dict, False) - self.debug = True + super().__init__(name_to_timeout_dict) self.feed_site = feed_site self.feed_uris = feed_uris self.page_title = page_title @@ -35,10 +35,6 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): self.details = grab_bag.grab_bag() self.filter = profanity_filter.ProfanityFilter() - @abstractmethod - def debug_prefix(self) -> str: - pass - @abstractmethod def get_headlines_page_prefix(self) -> str: pass @@ -136,7 +132,7 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): headlines.set_title("%s" % self.page_title) subset = self.news.subset(4) if subset is None: - self.debug_print("Not enough messages to choose from.") + logger.warning('Not enough messages to select from in shuffle_news?!') return False for msg in subset: headlines.add_item(msg) @@ -187,10 +183,11 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): } """ ) - details.set_title(f"{self.page_title}") + details.set_title(self.page_title) subset = self.details.subset(1) if subset is None: - self.debug_print("Not enough details to choose from.") + logger.warning('Not enough details to choose from in do_details') + logger.debug("Not enough details to choose from.") return False for msg in subset: blurb = msg @@ -209,47 +206,55 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): http.client.HTTPSConnection]] = None for uri in self.feed_uris: + url = None if self.should_use_https(): - self.debug_print("Fetching: https://%s%s" % (self.feed_site, uri)) + url = f'https://{self.feed_site}{uri}' + logger.info(f'Fetching: {url}') self.conn = http.client.HTTPSConnection(self.feed_site, timeout=10) else: - self.debug_print("Fetching: http://%s%s" % (self.feed_site, uri)) + url = f'http://{self.feed_site}{uri}' + logger.info(f'Fetching: {url}') self.conn = http.client.HTTPConnection(self.feed_site, timeout=10) - assert(self.conn is not None) + assert self.conn is not None + assert url is not None self.conn.request( "GET", uri, None, { "Accept": "*/*", -# "Cache-control": "max-age=50", -# "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Cache-control": "max-age=50", }, ) try: response = self.conn.getresponse() except Exception as e: - traceback.print_exc(file=sys.stdout) - print( - f"Exception in generic RSS renderer HTTP connection fetching {self.feed_site}{uri}" + logger.exception(e) + logger.error( + f"Exception in generic RSS renderer HTTP connection fetching {url}; giving up." ) return False if response.status != 200: - print( - f"{self.page_title}: RSS fetch_news error, response: {response.status}" + logger.error( + f'Unexpected status {response.status} while fetching {url}; giving up.' ) - self.debug_print(str(response.read())) return False - rss = ET.fromstring(response.read()) + raw = response.read() + logger.info(f'Status 200: got {len(raw)} bytes back from {url}') + rss = ET.fromstring(raw) channel = rss[0] title_filter = set() - for item in channel.getchildren(): + for item in list(channel): title = self.find_title(item) description = item.findtext("description") if title is not None: title = self.munge_title(title, item) + else: + logger.info('Skipping RSS feed item with no title.') + continue + logger.debug(f'Considering RSS item {title}...') if description is not None: description = self.munge_description(description, item) else: @@ -260,22 +265,22 @@ class generic_news_rss_renderer(renderer.debuggable_abstaining_renderer): link = item.findtext("link") if link is not None: link = self.munge_link(link) - - if title is None or not self.item_is_interesting_for_headlines( - title, description, item + if not self.item_is_interesting_for_headlines( + title, description, item ): - self.debug_print(f'Item "{title}" is not interesting') + logger.info(f'Skipping {title} because it\'s not interesting.') continue if self.should_profanity_filter() and ( self.filter.contains_bad_word(title) or self.filter.contains_bad_word(description) ): - self.debug_print(f'Found bad words in item "{title}"') + logger.info(f'Skipping {title} because it contains profanity.') continue if title in title_filter: - self.debug_print(f'Already saw title {title}, skipping.') + logger.info(f'Skipping {title} because we already saw an item with the same title.') + continue title_filter.add(title) blurb = """
' if self.item_is_interesting_for_article(title, description, item): + logger.info(f'Item {title} is also interesting as an article details page; creating...') longblurb = blurb longblurb += "
" longblurb += description longblurb += "
" longblurb = longblurb.replace("font-size:34pt", "font-size:44pt") self.details.add(longblurb) + else: + logger.info(f'Item {title} isn\'t interesting for article details page; skipped.') blurb += "" self.news.add(blurb) count += 1 + logger.debug(f'Added {count} items so far...') return count > 0 diff --git a/gkeep_renderer.py b/gkeep_renderer.py index d05a24b..b639ed2 100644 --- a/gkeep_renderer.py +++ b/gkeep_renderer.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -import gkeepapi # type: ignore +import logging import os import re -from typing import List, Dict +from typing import Dict -from google_auth_oauthlib.flow import InstalledAppFlow +import gkeepapi # type: ignore import constants import file_writer @@ -13,9 +13,12 @@ import renderer import kiosk_secrets as secrets -class gkeep_renderer(renderer.debuggable_abstaining_renderer): +logger = logging.getLogger(__file__) + + +class gkeep_renderer(renderer.abstaining_renderer): def __init__(self, name_to_timeout_dict: Dict[str, int]) -> None: - super(gkeep_renderer, self).__init__(name_to_timeout_dict, True) + super().__init__(name_to_timeout_dict) self.colors_by_name = { "white": "#002222", "green": "#345920", @@ -35,9 +38,9 @@ class gkeep_renderer(renderer.debuggable_abstaining_renderer): secrets.google_keep_username, secrets.google_keep_password ) if success: - self.debug_print("Connected with gkeep.") + logger.debug("Connected with gkeep.") else: - self.debug_print("Error connecting with gkeep.") + logger.debug("Error connecting with gkeep.") def debug_prefix(self) -> str: return "gkeep" @@ -55,10 +58,10 @@ class gkeep_renderer(renderer.debuggable_abstaining_renderer): filename = f"{title}_2_3600.html" contents = note.text + "\n" - self.debug_print(f"Note title '{title}'") + logger.debug(f"Note title '{title}'") if contents != "" and not contents.isspace(): contents = strikethrough.sub("", contents) - self.debug_print(f"Note contents:\n{contents}") + logger.debug(f"Note contents:\n{contents}") contents = contents.replace( "\u2610 ", '
  •  ' ) @@ -76,7 +79,7 @@ class gkeep_renderer(renderer.debuggable_abstaining_renderer): leading_spaces //= 2 leading_spaces = int(leading_spaces) x = x.lstrip(" ") - # self.debug_print(" * (%d) '%s'" % (leading_spaces, x)) + # logger.debug(" * (%d) '%s'" % (leading_spaces, x)) for y in range(0, leading_spaces): x = "