From 5c39d86ebc075ccb7be98b1dfab8040b72ff9134 Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Sat, 6 Nov 2021 14:15:49 -0700 Subject: [PATCH] Changes ;) --- bellevue_city_calendar_renderer.py | 90 +++++ bellevue_reporter_rss_renderer.py | 38 +- camera_trigger.py | 9 +- chooser.py | 2 +- cnn_rss_renderer.py | 2 +- constants.py | 1 + file_writer.py | 16 +- gcal_renderer.py | 157 +++++--- gdata_oauth.py | 445 ++++++++++++----------- generic_news_rss_renderer.py | 178 +++++----- gkeep_renderer.py | 4 +- google_news_rss_renderer.py | 7 +- grab_bag.py | 8 +- kiosk.py | 99 +++++- secrets.py => kiosk_secrets.py | 0 listen.py | 12 +- local_photos_mirror_renderer.py | 7 +- main.py | 553 +++++++++++++++++++++++++++++ myq_renderer.py | 10 +- pf.py | 452 +++++++++++++++++++++++ reddit_renderer.py | 14 +- renderer_catalog.py | 37 +- stock_renderer.py | 5 +- stranger_renderer.py | 26 +- test_listen.py | 27 ++ twitter_renderer.py | 6 +- urbanist_renderer.py | 68 ++++ weather_renderer.py | 4 +- 28 files changed, 1847 insertions(+), 430 deletions(-) create mode 100644 bellevue_city_calendar_renderer.py rename secrets.py => kiosk_secrets.py (100%) create mode 100755 main.py create mode 100644 pf.py create mode 100755 test_listen.py create mode 100644 urbanist_renderer.py diff --git a/bellevue_city_calendar_renderer.py b/bellevue_city_calendar_renderer.py new file mode 100644 index 0000000..4587569 --- /dev/null +++ b/bellevue_city_calendar_renderer.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +import datetime +import re +from typing import Dict, List, Optional +import xml + +from dateutil.parser import parse + +import generic_news_rss_renderer as gnrss + + +class bellevue_city_calendar_renderer(gnrss.generic_news_rss_renderer): + """Read the Bellevue City Calendar feed.""" + + def __init__( + self, + name_to_timeout_dict: Dict[str, int], + feed_site: str, + feed_uris: List[str], + page_title: str, + ): + super(bellevue_city_calendar_renderer, self).__init__( + name_to_timeout_dict, feed_site, feed_uris, page_title + ) + self.debug = True + + def debug_prefix(self) -> str: + return f"bellevue_calendar({self.page_title})" + + def get_headlines_page_prefix(self) -> str: + return "bellevue-calendar" + + def get_details_page_prefix(self) -> str: + return "bellevue-calendar-details" + + def should_use_https(self) -> bool: + return True + + def get_event_time(self, item: xml.etree.ElementTree.Element) -> Optional[datetime.datetime]: + return parse(self.find_pubdate(item)) + + def find_pubdate(self, item: xml.etree.ElementTree.Element) -> Optional[str]: + descr = item.findtext("description") + if descr is None: + return None + m = re.search(r'time datetime="([^"]+)"', descr) + if m is None: + return None + return m.group(1) + + def item_is_interesting_for_headlines( + self, title: str, description: str, item: xml.etree.ElementTree.Element + ) -> bool: + if "City Council" in title: + return False + if "City Offices Closed" in title: + return False + if "Regular Meeting" in title: + return False + if "Commission" in title: + return False + date = self.get_event_time(item) + if date is None: + return False + tzinfo = date.tzinfo + now = datetime.datetime.now(tzinfo) + delta = (now - date).total_seconds() / (60 * 60 * 24) + return delta < 0 + + def do_details(self) -> bool: + return False + + def item_is_interesting_for_article( + self, title: str, description: str, item: xml.etree.ElementTree.Element + ) -> bool: + return False + + +# Test +#x = bellevue_city_calendar_renderer( +# {"Fetch News" : 1, +# "Shuffle News" : 1}, +# "bellevuewa.gov", +# [ "/calendar/events.xml" ], +# "Test" ) +#if x.fetch_news() == 0: +# print("Error fetching news, no items fetched.") +#else: +# x.shuffle_news() diff --git a/bellevue_reporter_rss_renderer.py b/bellevue_reporter_rss_renderer.py index 104147d..b8fd27b 100644 --- a/bellevue_reporter_rss_renderer.py +++ b/bellevue_reporter_rss_renderer.py @@ -3,6 +3,7 @@ import re from typing import List, Dict import xml +import xml.etree.ElementTree as ET import generic_news_rss_renderer as gnrss @@ -23,7 +24,7 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): self.debug = True def debug_prefix(self) -> str: - return "bellevue_reporter(%s)" % (self.page_title) + return f"bellevue_reporter({self.page_title})" def get_headlines_page_prefix(self) -> str: return "bellevue-reporter" @@ -34,7 +35,7 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): def should_use_https(self) -> bool: return True - def munge_description(self, description: str) -> str: + def munge_description(self, description: str, item: ET.Element) -> str: description = re.sub("<[^>]+>", "", description) description = re.sub( "Bellevue\s+Reporter\s+Bellevue\s+Reporter", "", description @@ -56,31 +57,52 @@ class bellevue_reporter_rss_renderer(gnrss.generic_news_rss_renderer): def looks_like_review(title: str, description: str) -> bool: return "review" in title or "Review" in title + @staticmethod + def looks_like_spam(title: str, description: str) -> bool: + return ( + 'marketplace' in description + or 'national-marketplace' in description + or re.search('[Ww]eed', title) is not None + or re.search('[Cc]annabis', title) is not None + or re.search('[Cc]annabis', description) is not None + or 'THC' in title + or re.search('[Ll]ose [Ww]eight', title) is not None + or re.search('[Ll]ose [Ww]eight', description) is not None + ) + def item_is_interesting_for_headlines( self, title: str, description: str, item: xml.etree.ElementTree.Element ) -> bool: + unfiltered_description = item.findtext("description") if self.is_item_older_than_n_days(item, 10): - self.debug_print("%s: is too old!" % title) + self.debug_print(f'{title}: is too old!') + return False + if bellevue_reporter_rss_renderer.looks_like_spam(title, unfiltered_description): + self.debug_print(f'{title}: looks like spam') return False if bellevue_reporter_rss_renderer.looks_like_football(title, description): - self.debug_print("%s: looks like it's about football." % title) + self.debug_print(f'{title}: looks like it\'s about football.') return False if bellevue_reporter_rss_renderer.looks_like_review(title, description): - self.debug_print("%s: looks like bullshit." % title) + self.debug_print(f'{title}: looks like a review.') return False return True def item_is_interesting_for_article( self, title: str, description: str, item: xml.etree.ElementTree.Element ) -> bool: + unfiltered_description = item.findtext("description") if self.is_item_older_than_n_days(item, 10): - self.debug_print("%s: is too old!" % title) + self.debug_print(f'{title}: is too old!') + return False + if bellevue_reporter_rss_renderer.looks_like_spam(title, unfiltered_description): + self.debug_print(f'{title}: looks like spam') return False if bellevue_reporter_rss_renderer.looks_like_football(title, description): - self.debug_print("%s: looks like it's about football." % title) + self.debug_print(f'{title}: looks like it\'s about football.') return False if bellevue_reporter_rss_renderer.looks_like_review(title, description): - self.debug_print("%s: looks like bullshit." % title) + self.debug_print(f'{title}: looks like a review.') return False return True diff --git a/camera_trigger.py b/camera_trigger.py index b47a26e..0e9ac6d 100644 --- a/camera_trigger.py +++ b/camera_trigger.py @@ -15,20 +15,23 @@ class any_camera_trigger(trigger.trigger): self.triggers_in_the_past_seven_min = { "driveway": 0, "frontdoor": 0, + "doorbell": 0, "cabin_driveway": 0, } self.last_trigger_timestamp = { "driveway": 0, "frontdoor": 0, + "doorbell": 0, "cabin_driveway": 0, } def choose_priority(self, camera: str, age: int) -> int: """Based on the camera name and last trigger age, compute priority.""" base_priority_by_camera = { - "driveway": 1, + "driveway": 3, "frontdoor": 2, - "cabin_driveway": 1, + "doorbell": 1, + "cabin_driveway": 3, } priority = base_priority_by_camera[camera] if age < 10: @@ -43,7 +46,7 @@ class any_camera_trigger(trigger.trigger): """Return a list of triggered pages with priorities.""" triggers = [] num_cameras_with_recent_triggers = 0 - camera_list = ["driveway", "frontdoor", "cabin_driveway"] + camera_list = ["driveway", "frontdoor", "doorbell", "cabin_driveway"] now = time.time() try: diff --git a/chooser.py b/chooser.py index 3fecc09..beffdb2 100644 --- a/chooser.py +++ b/chooser.py @@ -151,7 +151,7 @@ class weighted_random_chooser_with_triggers(weighted_random_chooser): return (page, triggered) # Always show the clock in the middle of the night. - elif now.hour < 7: + elif now.hour < 6: for page in self.pages: if "clock" in page: return (page, False) diff --git a/cnn_rss_renderer.py b/cnn_rss_renderer.py index 9bec2a8..3e15c98 100644 --- a/cnn_rss_renderer.py +++ b/cnn_rss_renderer.py @@ -28,7 +28,7 @@ class cnn_rss_renderer(generic_news_rss_renderer.generic_news_rss_renderer): def get_details_page_prefix(self) -> str: return f"cnn-details-{self.page_title}" - def munge_description(self, description: str) -> str: + def munge_description(self, description: str, item: xml.etree.ElementTree.Element) -> str: description = re.sub("[Rr]ead full story for latest details.", "", description) description = re.sub("<[^>]+>", "", description) return description diff --git a/constants.py b/constants.py index 3463ea6..4dc8521 100644 --- a/constants.py +++ b/constants.py @@ -12,4 +12,5 @@ seconds_per_hour = seconds_per_minute * 60 seconds_per_day = seconds_per_hour * 24 myq_pagename = "myq_4_300.html" +internal_stats_pagename = 'internal_stats_1_1000.html' gcal_imminent_pagename = "hidden/gcal-imminent_0_none.html" diff --git a/file_writer.py b/file_writer.py index 3cb2f39..fced449 100644 --- a/file_writer.py +++ b/file_writer.py @@ -2,12 +2,15 @@ import constants import os +from uuid import uuid4 class file_writer: """Helper context to write a pages file.""" def __init__(self, filename: str, *, transformations=[]): + temp = "temp-" + str(uuid4()) + self.temp_filename = os.path.join(constants.pages_dir, temp) self.full_filename = os.path.join(constants.pages_dir, filename) self.xforms = [file_writer.remove_tricky_unicode] self.xforms.extend(transformations) @@ -29,11 +32,14 @@ class file_writer: self.f.write(data.encode("utf-8")) def __enter__(self): - self.f = open(self.full_filename, "wb") + self.f = open(self.temp_filename, "wb") return self def __exit__(self, exc_type, exc_value, exc_traceback): self.close() + cmd = f'/bin/mv -f {self.temp_filename} "{self.full_filename}"' + os.system(cmd) + print(cmd) def done(self): self.close() @@ -43,8 +49,8 @@ class file_writer: # Test -# def toupper(x): +#def toupper(x): # return x.upper() -# -# with file_writer("test", transformations=[toupper]) as fw: -# fw.write(u"Another test!!") + +#with file_writer("test", transformations=[toupper]) as fw: +# fw.write(u"Another test!!") diff --git a/gcal_renderer.py b/gcal_renderer.py index 11f5304..c43a144 100644 --- a/gcal_renderer.py +++ b/gcal_renderer.py @@ -4,18 +4,22 @@ contents of several Google calendars.""" import datetime +import functools +import os +import time +from typing import Any, Dict, List, Optional, Tuple + +from dateutil.parser import parse import gdata # type: ignore import gdata_oauth from oauth2client.client import AccessTokenRefreshError # type: ignore -import os -import time -from typing import Dict, List, Optional, Tuple +import pytz import constants import file_writer import globals import renderer -import secrets +import kiosk_secrets as secrets class gcal_renderer(renderer.debuggable_abstaining_renderer): @@ -34,6 +38,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): ] ) + @functools.total_ordering class comparable_event(object): """A helper class to sort events.""" @@ -46,6 +51,10 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): ) -> None: if start_time is None: assert(end_time is None) + else: + assert(isinstance(start_time, datetime.datetime)) + if end_time is not None: + assert(isinstance(end_time, datetime.datetime)) self.start_time = start_time self.end_time = end_time self.summary = summary @@ -63,7 +72,15 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): that.calendar, ) - def __str__(self) -> str: + def __eq__(self, that) -> bool: + return ( + self.start_time == that.start_time and + self.end_time == that.end_time and + self.summary == that.summary and + self.calendar == that.calendar + ) + + def __repr__(self) -> str: return "[%s] %s" % (self.timestamp(), self.friendly_name()) def friendly_name(self) -> str: @@ -72,14 +89,36 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): return "%s" % name def timestamp(self) -> str: + now = datetime.datetime.now(pytz.timezone("US/Pacific")) if self.start_time is None: return "None" - elif self.start_time.hour == 0: - return datetime.datetime.strftime(self.start_time, "%a %b %d %Y") + elif ( + self.start_time.hour == 0 and + self.start_time.minute == 0 and + self.start_time.second == 0 + ): + if self.start_time.year == now.year: + return datetime.datetime.strftime( + self.start_time, + "%a %b %d" + ) + else: + return datetime.datetime.strftime( + self.start_time, + "%a %b %d, %Y" + ) else: - return datetime.datetime.strftime( - self.start_time, "%a %b %d %Y %H:%M%p" - ) + dt = self.start_time + zone = dt.tzinfo + local_dt = dt.replace(tzinfo=zone).astimezone(tz=pytz.timezone('US/Pacific')) + if local_dt.year == now.year: + return datetime.datetime.strftime( + local_dt, "%a %b %d %I:%M%p" + ) + else: + return datetime.datetime.strftime( + local_dt, "%a %b %d, %Y %I:%M%p" + ) def __init__( self, name_to_timeout_dict: Dict[str, int], oauth: gdata_oauth.OAuth @@ -103,9 +142,9 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): raise Exception("Unexpected operation") def get_min_max_timewindow(self) -> Tuple[str, str]: - now = datetime.datetime.now() - _time_min = now - datetime.timedelta(1) - _time_max = now + datetime.timedelta(95) + now = datetime.datetime.now(pytz.timezone("US/Pacific")) + _time_min = now - datetime.timedelta(hours=6) + _time_max = now + datetime.timedelta(days=95) time_min = datetime.datetime.strftime(_time_min, "%Y-%m-%dT%H:%M:%SZ") time_max = datetime.datetime.strftime(_time_max, "%Y-%m-%dT%H:%M:%SZ") self.debug_print(f"time_min is {time_min}") @@ -113,19 +152,22 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): return (time_min, time_max) @staticmethod - def parse_date(date_str: str) -> Optional[datetime.datetime]: - retval = None - try: - _ = date_str.get("date") - if _: - retval = datetime.datetime.strptime(_, "%Y-%m-%d") - else: - _ = date_str.get("dateTime") - if _: - retval = datetime.datetime.strptime(_[:-6], "%Y-%m-%dT%H:%M:%S") - return retval - except: - pass + def parse_date(date: Any) -> Optional[datetime.datetime]: + if isinstance(date, datetime.datetime): + return date + elif isinstance(date, dict): + if 'dateTime' in date: + d = date['dateTime'] + dt = parse(d) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=None).astimezone(tz=pytz.timezone('US/Pacific')) + return dt + elif 'date' in date: + d = date['date'] + dt = datetime.datetime.strptime(d, '%Y-%m-%d') + dt = dt.replace(tzinfo=None).astimezone(tz=pytz.timezone('US/Pacific')) + return dt + print(f'Not sure what to do with this {date} ({type(date)}), help?!') return None def get_events_from_interesting_calendars( @@ -156,12 +198,13 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): ) for event in events["items"]: summary = event["summary"] - self.debug_print( - f" ... event '{summary}' ({event['start']} to {event['end']}" - ) start = gcal_renderer.parse_date(event["start"]) end = gcal_renderer.parse_date(event["end"]) + self.debug_print( + f" ... event '{summary}' ({event['start']} ({start}) to {event['end']} ({end})" + ) if start is not None and end is not None: + self.debug_print(f' ... adding {summary} to sortable_events') sortable_events.append( gcal_renderer.comparable_event( start, end, summary, calendar["summary"] @@ -172,7 +215,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): or "Holidays" in calendar["summary"] or "Countdown" in summary ): - self.debug_print(" ... event is countdown worthy!") + self.debug_print(f" ... adding {summary} to countdown_events") countdown_events.append( gcal_renderer.comparable_event( start, end, summary, calendar["summary"] @@ -193,20 +236,32 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): ) = self.get_events_from_interesting_calendars(time_min, time_max) self.sortable_events.sort() with file_writer.file_writer("gcal_3_86400.html") as f: - f.write("

Upcoming Calendar Events:


\n") - f.write("
\n") + f.write( +f""" +

Upcoming Calendar Events:

+
+
+
+""" + ) upcoming_sortable_events = self.sortable_events[:12] - for event in upcoming_sortable_events: + for n, event in enumerate(upcoming_sortable_events): + self.debug_print(f'{n}/12: {event.friendly_name()} / {event.calendar}') + if n % 2 == 0: + color = "#c6b0b0" + else: + color = "#eeeeee" f.write( - f""" - - - -\n""" +f""" + + + + +""" ) f.write("
- {event.timestamp()} - - {event.friendly_name()} -
+ {event.timestamp()} + + {event.friendly_name()} +
\n") @@ -214,7 +269,7 @@ class gcal_renderer(renderer.debuggable_abstaining_renderer): self.countdown_events.sort() with file_writer.file_writer("countdown_3_7200.html") as g: g.write("

Countdowns: