From d6990436e08a57ce211b10058dc61fb223cb94ec Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Sat, 28 Nov 2020 13:25:21 -0800 Subject: [PATCH] Various changes --- bellevue_reporter_rss_renderer.py | 12 ++++++ chooser.py | 66 ++++++++++++++++++++++++------- constants.py | 2 - gcal_renderer.py | 7 ++-- generic_news_rss_renderer.py | 19 ++++++--- gkeep_renderer.py | 18 ++++----- google_news_rss_renderer.py | 18 +++++++-- health_renderer.py | 23 ++++++++--- kiosk.py | 25 +++++++++--- local_photos_mirror_renderer.py | 65 +++++++++++++++++++----------- myq_renderer.py | 8 ++-- profanity_filter.py | 2 + reddit_renderer.py | 1 + renderer.py | 4 ++ renderer_catalog.py | 2 +- seattletimes_rss_renderer.py | 1 + stevens_renderer.py | 2 +- stranger_renderer.py | 3 +- twitter_renderer.py | 16 ++++---- 19 files changed, 207 insertions(+), 87 deletions(-) diff --git a/bellevue_reporter_rss_renderer.py b/bellevue_reporter_rss_renderer.py index b71a34b..78ec694 100644 --- a/bellevue_reporter_rss_renderer.py +++ b/bellevue_reporter_rss_renderer.py @@ -33,12 +33,24 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): if self.is_item_older_than_n_days(item, 10): self.debug_print("%s: is too old!" % title) return False + if (title.find("NFL") != -1 or + re.search("[Ll]ive [Ss]tream", title) != None or + re.search("[Ll]ive[Ss]tream", title) != None or + re.search("[Ll]ive [Ss]tream", description) != None): + self.debug_print("%s: looks like it's about football." % title) + return False return True def item_is_interesting_for_article(self, title, description, item): if self.is_item_older_than_n_days(item, 10): self.debug_print("%s: is too old!" % title) return False + if (title.find(" NFL") != -1 or + re.search("[Ll]ive [Ss]tream", title) != None or + re.search("[Ll]ive[Ss]tream", title) != None or + re.search("[Ll]ive [Ss]tream", description) != None): + self.debug_print("%s: looks like it's about football." % title) + return False return True # Test diff --git a/chooser.py b/chooser.py index df662da..9bf98e3 100644 --- a/chooser.py +++ b/chooser.py @@ -1,3 +1,4 @@ +import datetime import os import random import re @@ -34,12 +35,22 @@ class chooser(object): pass class weighted_random_chooser(chooser): - """Chooser that does it via weighted RNG""" - def __init__(self): + """Chooser that does it via weighted RNG.""" + def dont_choose_page_twice_in_a_row_filter(self, choice): + if choice == self.last_choice: + return False + self.last_choice = choice + return True + + def __init__(self, filter_list): self.last_choice = "" self.valid_filename = re.compile("([^_]+)_(\d+)_([^\.]+)\.html") self.pages = None self.count = 0 + self.filter_list = filter_list + if filter_list is None: + self.filter_list = [] + self.filter_list.append(self.dont_choose_page_twice_in_a_row_filter) def choose_next_page(self): if (self.pages == None or @@ -54,26 +65,39 @@ class weighted_random_chooser(chooser): weight = int(result.group(2)) weights.append(weight) total_weight += weight - if (total_weight <= 0): raise error while True: - pick = random.randrange(0, total_weight - 1) + random_pick = random.randrange(0, total_weight - 1) so_far = 0 for x in range(0, len(weights)): so_far += weights[x] - if (so_far > pick and - self.pages[x] != self.last_choice): - self.last_choice = self.pages[x] - self.count += 1 - return self.pages[x] + if so_far > random_pick: + break + choice = self.pages[x] + + # Allow filter list to suppress pages. + choice_is_filtered = False + for f in self.filter_list: + if not f(choice): + print("chooser: %s filtered by %s" % (choice, f.__name__)) + choice_is_filtered = True + break + if choice_is_filtered: + continue + + # We're good... + self.count += 1 + return choice class weighted_random_chooser_with_triggers(weighted_random_chooser): """Same as WRC but has trigger events""" - def __init__(self, trigger_list): - weighted_random_chooser.__init__(self) + def __init__(self, trigger_list, filter_list): + weighted_random_chooser.__init__(self, filter_list) self.trigger_list = trigger_list + if trigger_list is None: + self.trigger_list = [] self.page_queue = set(()) def check_for_triggers(self): @@ -93,9 +117,9 @@ class weighted_random_chooser_with_triggers(weighted_random_chooser): triggered = self.check_for_triggers() - # First try to satisfy from the page queue + # First try to satisfy from the page queue. if (len(self.page_queue) > 0): - print("Pulling page from queue") + print("chooser: Pulling page from queue...") page = None priority = None for t in self.page_queue: @@ -133,4 +157,18 @@ class rotating_chooser(chooser): self.count += 1 return page -#x = weighted_random_chooser_with_triggers(None) +# Test +def filter_news_during_dinnertime(page): + now = datetime.datetime.now() + is_dinnertime = now.hour >= 17 and now.hour <= 20 + return (not is_dinnertime or + not ("cnn" in page or + "news" in page or + "mynorthwest" in page or + "seattle" in page or + "stranger" in page or + "twitter" in page or + "wsj" in page)) + +#x = weighted_random_chooser_with_triggers([], [ filter_news_during_dinnertime ]) +#print(x.choose_next_page()) diff --git a/constants.py b/constants.py index 880fb85..3dfa4a3 100644 --- a/constants.py +++ b/constants.py @@ -1,5 +1,3 @@ -#!/usr/local/bin/python - refresh_period_sec = 22 render_period_sec = 30 pages_dir = "/usr/local/export/www/kiosk/pages" diff --git a/gcal_renderer.py b/gcal_renderer.py index c3be3d7..a248d1d 100644 --- a/gcal_renderer.py +++ b/gcal_renderer.py @@ -86,8 +86,9 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): page_token = None def format_datetime(x): return datetime.datetime.strftime(x, '%Y-%m-%dT%H:%M:%SZ') - time_min = datetime.datetime.now() - time_max = time_min + datetime.timedelta(95) + now = datetime.datetime.now() + time_min = now - datetime.timedelta(1) + time_max = now + datetime.timedelta(95) time_min, time_max = list(map(format_datetime, (time_min, time_max))) self.debug_print("time_min is %s" % time_min) self.debug_print("time_max is %s" % time_max) @@ -95,7 +96,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): # Writes 2 files: # + "upcoming events", # + a countdown timer for a subser of events, - f = file_writer.file_writer('gcal_3_none.html') + f = file_writer.file_writer('gcal_3_86400.html') f.write('

Upcoming Calendar Events:

\n') f.write('
\n') diff --git a/generic_news_rss_renderer.py b/generic_news_rss_renderer.py index 5cf6e6c..698f7aa 100644 --- a/generic_news_rss_renderer.py +++ b/generic_news_rss_renderer.py @@ -127,7 +127,7 @@ a:active { font-weight: bold; } """) - f = file_writer.file_writer('%s_%s_none.html' % ( + f = file_writer.file_writer('%s_%s_25900.html' % ( self.get_headlines_page_prefix(), self.get_headlines_page_priority())) headlines.render_html(f) @@ -162,7 +162,7 @@ a:active { blurb = msg blurb += u'' details.add_item(blurb) - g = file_writer.file_writer('%s_%s_none.html' % ( + g = file_writer.file_writer('%s_%s_86400.html' % ( self.get_details_page_prefix(), self.get_details_page_priority())) details.render_html(g) @@ -177,16 +177,23 @@ a:active { for uri in self.feed_uris: if self.should_use_https(): self.debug_print("Fetching: https://%s%s" % (self.feed_site, uri)) - self.conn = http.client.HTTPSConnection(self.feed_site) + self.conn = http.client.HTTPSConnection(self.feed_site, timeout=20) else: self.debug_print("Fetching: http://%s%s" % (self.feed_site, uri)) - self.conn = http.client.HTTPConnection(self.feed_site) + self.conn = http.client.HTTPConnection(self.feed_site, timeout=20) self.conn.request( "GET", uri, None, - {"Accept-Charset": "utf-8"}) - response = self.conn.getresponse() + { "Accept": "*/*", + "Cache-control": "max-age=59", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36"}) + try: + response = self.conn.getresponse() + except: + print("Exception in generic RSS renderer HTTP connection") + return False + if response.status != 200: print(("%s: RSS fetch_news error, response: %d" % (self.page_title, response.status))) diff --git a/gkeep_renderer.py b/gkeep_renderer.py index f8313bd..de1116d 100644 --- a/gkeep_renderer.py +++ b/gkeep_renderer.py @@ -52,10 +52,9 @@ class gkeep_renderer(renderer.debuggable_abstaining_renderer): self.debug_print("Note title '%s'" % title) if contents != '' and not contents.isspace(): contents = strikethrough.sub('', contents) + self.debug_print("Note contents:\n%s" % contents) contents = contents.replace(u'\u2610 ', u'
  •  ') - - #self.debug_print("Note contents:\n%s" % contents) contents = linkify.sub(r'\1', contents) individual_lines = contents.split("\n") @@ -66,13 +65,14 @@ class gkeep_renderer(renderer.debuggable_abstaining_renderer): length = len(x) if length > max_length: max_length = length - spaces = len(x) - len(x.lstrip(' ')) - spaces /= 2 - spaces = int(spaces) + leading_spaces = len(x) - len(x.lstrip(' ')) + leading_spaces /= 2 + leading_spaces = int(leading_spaces) x = x.lstrip(' ') - for y in range(0, spaces): + # self.debug_print(" * (%d) '%s'" % (leading_spaces, x)) + for y in range(0, leading_spaces): x = "
      " + x - for y in range(0, spaces): + for y in range(0, leading_spaces): x = x + "
    " contents = contents + x + "\n" @@ -122,5 +122,5 @@ class gkeep_renderer(renderer.debuggable_abstaining_renderer): return True # Test -x = gkeep_renderer({"Test", 1234}) -x.periodic_render("Test") +#x = gkeep_renderer({"Test", 1234}) +#x.periodic_render("Test") diff --git a/google_news_rss_renderer.py b/google_news_rss_renderer.py index 7ca37e8..b4290f3 100644 --- a/google_news_rss_renderer.py +++ b/google_news_rss_renderer.py @@ -27,15 +27,20 @@ class google_news_rss_renderer(generic_news_rss_renderer.generic_news_rss_render descr = descr + " (%s)" % source return descr + def munge_description_internal(self, descr): + if len(descr) > 450: + descr = descr[:450] + descr = re.sub(r"\<[^\>]*$", "", descr) + descr = descr + " [...]" + descr += "

    • " + return descr + def munge_description(self, description): soup = BeautifulSoup(description) for a in soup.findAll('a'): del a['href'] descr = str(soup) - if len(descr) > 400: - descr = descr[:400] - descr = descr + " [...]" - return descr + return munge_description_internal(descr) def find_image(self, item): return None @@ -59,3 +64,8 @@ class google_news_rss_renderer(generic_news_rss_renderer.generic_news_rss_render #if x.fetch_news() == 0: # print("Error fetching news, no items fetched.") #x.shuffle_news() +# +#descr = "this is a lot of really long text about nothign in particular. It's pretty interesting, don't you think? I hope that the munge description method works by both truncating it and remembering to close any open
  • items as well as making sure not to truncate in the middle of a
    • Out!" +#d = x.munge_description_internal(descr) +#print(d) + diff --git a/health_renderer.py b/health_renderer.py index 4eeba98..63f923f 100644 --- a/health_renderer.py +++ b/health_renderer.py @@ -17,7 +17,15 @@ class periodic_health_renderer(renderer.debuggable_abstaining_renderer): days = constants.seconds_per_day hours = constants.seconds_per_hour mins = constants.seconds_per_minute + minutes = mins limits = { + timestamps + 'last_http_probe_wannabe_house' : mins * 10, + timestamps + 'last_http_probe_meerkat_cabin' : mins * 10, + timestamps + 'last_http_probe_dns_house' : mins * 10, + timestamps + 'last_http_probe_rpi_cabin' : mins * 10, + timestamps + 'last_http_probe_rpi_house' : mins * 10, + timestamps + 'last_http_probe_therm_house' : mins * 10, + timestamps + 'last_rsnapshot_hourly' : hours * 24, timestamps + 'last_rsnapshot_daily' : days * 3, timestamps + 'last_rsnapshot_weekly' : days * 14, @@ -29,17 +37,22 @@ class periodic_health_renderer(renderer.debuggable_abstaining_renderer): timestamps + 'last_zfssnapshot_monthly' : days * 70, timestamps + 'last_zfssnapshot_cleanup' : hours * 24, - timestamps + 'last_disk_selftest_short' : days * 14, - timestamps + 'last_disk_selftest_long' : days * 31, timestamps + 'last_zfs_scrub' : days * 9, - timestamps + 'last_zfs_scrub_backup' : days * 9, + timestamps + 'last_backup_zfs_scrub' : days * 9, + timestamps + 'last_cabin_zfs_scrub' : days * 9, timestamps + 'last_zfsxfer_backup.house' : hours * 36, timestamps + 'last_zfsxfer_ski.dyn.guru.org' : days * 7, timestamps + 'last_photos_sync' : hours * 8, - timestamps + 'last_disk_selftest_backup_short': days * 14, - timestamps + 'last_disk_selftest_backup_long' : days * 31, + timestamps + 'last_disk_selftest_short' : days * 14, + timestamps + 'last_disk_selftest_long' : days * 31, + timestamps + 'last_backup_disk_selftest_short': days * 14, + timestamps + 'last_backup_disk_selftest_long' : days * 31, + timestamps + 'last_cabin_disk_selftest_short' : days * 14, + timestamps + 'last_cabin_disk_selftest_long' : days * 31, + + timestamps + 'last_cabin_rpi_ping' : mins * 10, timestamps + 'last_healthy_wifi' : mins * 10, timestamps + 'last_healthy_network' : mins * 10, timestamps + 'last_scott_sync' : days * 2, diff --git a/kiosk.py b/kiosk.py index 6003165..d9f607e 100755 --- a/kiosk.py +++ b/kiosk.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python3.7 +#!/usr/bin/env python3 import sys import traceback @@ -15,9 +15,22 @@ import logging import trigger_catalog import utils +def filter_news_during_dinnertime(page): + now = datetime.now() + is_dinnertime = now.hour >= 17 and now.hour <= 20 + return (not is_dinnertime or + not ("cnn" in page or + "news" in page or + "mynorthwest" in page or + "seattle" in page or + "stranger" in page or + "twitter" in page or + "wsj" in page)) + def thread_change_current(): page_chooser = chooser.weighted_random_chooser_with_triggers( - trigger_catalog.get_triggers()) + trigger_catalog.get_triggers(), + [ filter_news_during_dinnertime ]) swap_page_target = 0 last_page = "" while True: @@ -246,13 +259,14 @@ def emit_wrapped(f, filename):

    +

    - %s @ %s ago. + %s @ %s ago.

    +
    diff --git a/local_photos_mirror_renderer.py b/local_photos_mirror_renderer.py index 32e0c1e..0b8f7fc 100644 --- a/local_photos_mirror_renderer.py +++ b/local_photos_mirror_renderer.py @@ -2,6 +2,7 @@ import os import file_writer import renderer import random +import re class local_photos_mirror_renderer(renderer.debuggable_abstaining_renderer): """A renderer that uses a local mirror of Google photos""" @@ -9,28 +10,38 @@ class local_photos_mirror_renderer(renderer.debuggable_abstaining_renderer): album_root_directory = "/usr/local/export/www/gphotos/albums" album_whitelist = frozenset([ - '1208 Newer Alex Photos', - '1013 Scott and Lynn', - '0106 Key West 2019', - '1017 Olympic Sculpture Park', - '0212 Chihuly Glass', - '0730 Trip to East Coast \'16', - '0715 Barn', - '1009 East Coast 2018', - '0819 Skiing with Alex', - '0819 Friends', - '0227 Trip to California, \'16', - '0407 London, 2018', - '0528 Ohme Gardens', - '0809 Bangkok and Phuket, 2003', - '0803 Blue Angels... Seafair', - '0719 Dunn Gardens', - '0514 Krakow 2009', - '0515 Tuscany 2008', - '0508 Yosemite 2010', - '0611 Sonoma', - '1025 NJ 2015', - '0407 Las Vegas, 2017', + '8-Mile Lake Hike', + 'Bangkok and Phuket, 2003', + 'Barn', + 'Blue Angels... Seafair', + 'Chihuly Glass', + 'Dunn Gardens', + 'East Coast 2018', + 'Fall \'17', + 'Friends', + 'Hiking', + 'Key West 2019', + 'Krakow 2009', + 'Kubota Gardens', + 'Las Vegas, 2017', + 'London, 2018', + 'Munich, July 2018', + 'NJ 2015', + 'Newer Alex Photos', + 'Ohme Gardens', + 'Olympic Sculpture Park', + 'Prague and Munich 2019', + 'Random', + 'Scott and Lynn', + 'SFO 2014', + 'Skiing with Alex', + 'Sonoma', + 'Trip to California, \'16', + 'Trip to San Francisco', + 'Trip to East Coast \'16', + 'Tuscany 2008', + 'Yosemite 2010', + 'Zoo', ]) extension_whitelist = frozenset([ @@ -56,12 +67,18 @@ class local_photos_mirror_renderer(renderer.debuggable_abstaining_renderer): else: raise error('Unexpected operation') + def album_is_in_whitelist(self, name): + for wlalbum in self.album_whitelist: + if re.search('\d+ %s' % wlalbum, name) != None: + return True + return False + # Walk the filesystem looking for photos in whitelisted albums and # keep their paths in memory. def index_photos(self): for root, subdirs, files in os.walk(self.album_root_directory): last_dir = root.rsplit('/', 1)[1] - if last_dir in self.album_whitelist: + if self.album_is_in_whitelist(last_dir): for x in files: extension = x.rsplit('.', 1)[1] if extension in self.extension_whitelist: @@ -79,7 +96,7 @@ class local_photos_mirror_renderer(renderer.debuggable_abstaining_renderer): print("No photos!") return False path = random.sample(self.candidate_photos, 1)[0] - f = file_writer.file_writer('photo_23_none.html') + f = file_writer.file_writer('photo_23_3600.html') f.write("""