From: Scott Date: Thu, 6 Jan 2022 20:13:34 +0000 (-0800) Subject: A bunch of changes... X-Git-Url: https://wannabe.guru.org/gitweb/?a=commitdiff_plain;h=5f75cf834725ac26b289cc5f157af0cb71cd5f0e;p=python_utils.git A bunch of changes... --- diff --git a/.gitignore b/.gitignore index 28e68dd..221b64b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +scott_secrets.py __pycache__/* */__pycache__/* .mypy_cache/* diff --git a/arper.py b/arper.py index 2171e77..8f419f9 100644 --- a/arper.py +++ b/arper.py @@ -48,7 +48,7 @@ cfg.add_argument( @persistent.persistent_autoloaded_singleton() class Arper(persistent.Persistent): def __init__( - self, cached_state: Optional[BiDict[str, str]] = None + self, cached_state: Optional[BiDict] = None ) -> None: self.state = BiDict() if cached_state is not None: diff --git a/base_presence.py b/base_presence.py new file mode 100755 index 0000000..cd62bb7 --- /dev/null +++ b/base_presence.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 + +import datetime +from collections import defaultdict +import logging +import re +from typing import Dict, List, Set + +# Note: this module is fairly early loaded. Be aware of dependencies. +import argparse_utils +import bootstrap +import config +from type.locations import Location +from type.people import Person +import site_config + + +logger = logging.getLogger(__name__) + +cfg = config.add_commandline_args( + f"Presence Detection ({__file__})", + "Args related to detection of human beings in locations.", +) +cfg.add_argument( + "--presence_macs_file", + type=argparse_utils.valid_filename, + default = "/home/scott/cron/persisted_mac_addresses.txt", + metavar="FILENAME", + help="The location of persisted_mac_addresses.txt to use." +) +cfg.add_argument( + '--presence_tolerable_staleness_seconds', + type=argparse_utils.valid_duration, + default=datetime.timedelta(seconds=60 * 5), + metavar='DURATION', + help='Max acceptable age of location data before auto-refreshing' +) + + +class PresenceDetection(object): + def __init__(self) -> None: + # Note: list most important devices first. + self.devices_by_person: Dict[Person, List[str]] = { + Person.SCOTT: [ + "3C:28:6D:10:6D:41", # pixel3 + "6C:40:08:AE:DC:2E", # laptop + ], + Person.LYNN: [ + "08:CC:27:63:26:14", # motog7 + "B8:31:B5:9A:4F:19", # laptop + ], + Person.ALEX: [ + "0C:CB:85:0C:8B:AE", # phone + "D0:C6:37:E3:36:9A", # laptop + ], + Person.AARON_AND_DANA: [ + "98:B6:E9:E5:5A:7C", + "D6:2F:37:CA:B2:9B", + "6C:E8:5C:ED:17:26", + "90:E1:7B:13:7C:E5", + "6E:DC:7C:75:02:1B", + "B2:16:1A:93:7D:50", + "18:65:90:DA:3A:35", + "22:28:C8:7D:3C:85", + "B2:95:23:69:91:F8", + "96:69:2C:88:7A:C3", + ], + } + self.run_location = site_config.get_location() + logger.debug(f"run_location is {self.run_location}") + self.weird_mac_at_cabin = False + self.location_ts_by_mac: Dict[ + Location, Dict[str, datetime.datetime] + ] = defaultdict(dict) + self.names_by_mac: Dict[str, str] = {} + self.dark_locations: Set[Location] = set() + self.last_update = None + + def maybe_update(self) -> None: + if self.last_update is None: + self.update() + else: + now = datetime.datetime.now() + delta = now - self.last_update + if delta.total_seconds() > config.config['presence_tolerable_staleness_seconds'].total_seconds(): + logger.debug( + f"It's been {delta.total_seconds()}s since last update; refreshing now." + ) + self.update() + + def update(self) -> None: + self.dark_locations = set() + if self.run_location is Location.HOUSE: + self.update_from_house() + elif self.run_location is Location.CABIN: + self.update_from_cabin() + else: + raise Exception("Where the hell is this running?!") + self.last_update = datetime.datetime.now() + + def update_from_house(self) -> None: + from exec_utils import cmd + try: + persisted_macs = config.config['presence_macs_file'] + except KeyError: + persisted_macs = '/home/scott/cron/persisted_mac_addresses.txt' + self.read_persisted_macs_file(persisted_macs, Location.HOUSE) + try: + raw = cmd( + "ssh scott@meerkat.cabin 'cat /home/scott/cron/persisted_mac_addresses.txt'", + timeout_seconds=10.0, + ) + self.parse_raw_macs_file(raw, Location.CABIN) + except Exception as e: + logger.exception(e) + logger.warning("Can't see the cabin right now; presence detection impared.") + self.dark_locations.add(Location.CABIN) + + def update_from_cabin(self) -> None: + from exec_utils import cmd + try: + persisted_macs = config.config['presence_macs_file'] + except KeyError: + persisted_macs = '/home/scott/cron/persisted_mac_addresses.txt' + self.read_persisted_macs_file(persisted_macs, Location.CABIN) + try: + raw = cmd( + "ssh scott@wennabe.house 'cat /home/scott/cron/persisted_mac_addresses.txt'", + timeout_seconds=10.0, + ) + self.parse_raw_macs_file(raw, Location.HOUSE) + except Exception as e: + logger.exception(e) + logger.warning("Can't see the house right now; presence detection impared.") + self.dark_locations.add(Location.HOUSE) + + def read_persisted_macs_file( + self, filename: str, location: Location + ) -> None: + if location is Location.UNKNOWN: + return + with open(filename, "r") as rf: + lines = rf.read() + self.parse_raw_macs_file(lines, location) + + def parse_raw_macs_file(self, raw: str, location: Location) -> None: + lines = raw.split("\n") + + # CC:F4:11:D7:FA:EE, 2240, 10.0.0.22 (side_deck_high_home), Google, 1611681990 + cabin_count = 0 + for line in lines: + line = line.strip() + if len(line) == 0: + continue + logger.debug(f'{location}> {line}') + if "cabin_" in line: + continue + if location == Location.CABIN: + logger.debug('Cabin count: {cabin_count}') + cabin_count += 1 + try: + (mac, count, ip_name, mfg, ts) = line.split(",") + except Exception as e: + logger.error(f'SKIPPED BAD LINE> {line}') + logger.exception(e) + continue + mac = mac.strip() + (self.location_ts_by_mac[location])[ + mac + ] = datetime.datetime.fromtimestamp(int(ts.strip())) + ip_name = ip_name.strip() + match = re.match(r"(\d+\.\d+\.\d+\.\d+) +\(([^\)]+)\)", ip_name) + if match is not None: + name = match.group(2) + self.names_by_mac[mac] = name + if cabin_count > 0: + logger.debug('Weird MAC at the cabin') + self.weird_mac_at_cabin = True + + def is_anyone_in_location_now(self, location: Location) -> bool: + self.maybe_update() + if location in self.dark_locations: + raise Exception(f"Can't see {location} right now; answer undefined.") + for person in Person: + if person is not None: + loc = self.where_is_person_now(person) + if location == loc: + return True + if location == location.CABIN and self.weird_mac_at_cabin: + return True + return False + + def where_is_person_now(self, name: Person) -> Location: + self.maybe_update() + if len(self.dark_locations) > 0: + logger.warning( + f"Can't see {self.dark_locations} right now; answer confidence impacted" + ) + logger.debug(f'Looking for {name}...') + + if name is Person.UNKNOWN: + if self.weird_mac_at_cabin: + return Location.CABIN + else: + return Location.UNKNOWN + + import dict_utils + votes: Dict[Location, int] = {} + tiebreaks: Dict[Location, datetime.datetime] = {} + credit = 10000 + for mac in self.devices_by_person[name]: + if mac not in self.names_by_mac: + continue + mac_name = self.names_by_mac[mac] + logger.debug(f'Looking for {name}... check for mac {mac} ({mac_name})') + for location in self.location_ts_by_mac: + if mac in self.location_ts_by_mac[location]: + ts = (self.location_ts_by_mac[location])[mac] + logger.debug(f'Seen {mac} ({mac_name}) at {location} since {ts}') + tiebreaks[location] = ts + + (most_recent_location, first_seen_ts) = dict_utils.item_with_max_value(tiebreaks) + bonus = credit + v = votes.get(most_recent_location, 0) + votes[most_recent_location] = v + bonus + logger.debug(f'{name}: {location} gets {bonus} votes.') + credit = int( + credit * 0.2 + ) # Note: list most important devices first + if credit <= 0: + credit = 1 + if len(votes) > 0: + (location, value) = dict_utils.item_with_max_value(votes) + if value > 2001: + return location + return Location.UNKNOWN + + +@bootstrap.initialize +def main() -> None: + p = PresenceDetection() + for person in Person: + print(f'{person} => {p.where_is_person_now(person)}') + print() + for location in Location: + print(f'{location} => {p.is_anyone_in_location_now(location)}') + + +if __name__ == '__main__': + main() diff --git a/cached/weather_forecast.py b/cached/weather_forecast.py index d1e7540..b343938 100644 --- a/cached/weather_forecast.py +++ b/cached/weather_forecast.py @@ -19,6 +19,8 @@ import datetime_utils import dateparse.dateparse_utils as dp import persistent import text_utils +import smart_home.thermometers as temps + logger = logging.getLogger(__name__) @@ -61,23 +63,9 @@ class CachedDetailedWeatherForecast(persistent.Persistent): self.forecasts = {} # Ask the raspberry pi about the outside temperature. - www = None - try: - www = urllib.request.urlopen( - "http://10.0.0.75/~pi/outside_temp", - timeout=2, - ) - current_temp = www.read().decode("utf-8") - current_temp = float(current_temp) - current_temp *= (9/5) - current_temp += 32.0 - current_temp = round(current_temp) - except Exception: - logger.warning('Timed out reading 10.0.0.75/~pi/outside_temp?!') - current_temp = None - finally: - if www is not None: - www.close() + current_temp = temps.ThermometerRegistry().read_temperature( + 'house_outside', convert_to_fahrenheit=True + ) # Get a weather forecast for Bellevue. www = urllib.request.urlopen( diff --git a/camera_utils.py b/camera_utils.py index e85bd6e..acf760d 100644 --- a/camera_utils.py +++ b/camera_utils.py @@ -12,6 +12,7 @@ import numpy as np import requests import decorator_utils +import exceptions logger = logging.getLogger(__name__) @@ -23,28 +24,35 @@ class RawJpgHsv(NamedTuple): hsv: Optional[np.ndarray] -class BlueIrisImageMetadata(NamedTuple): +class SanityCheckImageMetadata(NamedTuple): """Is a Blue Iris image bad (big grey borders around it) or infrared?""" is_bad_image: bool is_infrared_image: bool -def analyze_blue_iris_image(hsv: np.ndarray) -> BlueIrisImageMetadata: +def sanity_check_image(hsv: np.ndarray) -> SanityCheckImageMetadata: """See if a Blue Iris image is bad and infrared.""" + def is_near(a, b) -> bool: + return abs(a - b) < 3 + rows, cols, _ = hsv.shape num_pixels = rows * cols + weird_orange_count = 0 hs_zero_count = 0 - gray_count = 0 for r in range(rows): for c in range(cols): pixel = hsv[(r, c)] - if pixel[0] == 0 and pixel[1] == 0: + if ( + is_near(pixel[0], 16) and + is_near(pixel[1], 117) and + is_near(pixel[2], 196) + ): + weird_orange_count += 1 + elif (is_near(pixel[0], 0) and is_near(pixel[1], 0)): hs_zero_count += 1 - if abs(pixel[2] - 64) <= 10: - gray_count += 1 - logger.debug(f"gray#={gray_count}, hs0#={hs_zero_count}") - return BlueIrisImageMetadata( - gray_count > (num_pixels * 0.33), hs_zero_count > (num_pixels * 0.75) + logger.debug(f"hszero#={hs_zero_count}, weird_orange={weird_orange_count}") + return SanityCheckImageMetadata( + hs_zero_count > (num_pixels * 0.75), weird_orange_count > (num_pixels * 0.75) ) @@ -55,15 +63,19 @@ def fetch_camera_image_from_video_server( """Fetch the raw webcam image from the video server.""" camera_name = camera_name.replace(".house", "") camera_name = camera_name.replace(".cabin", "") - url = f"http://10.0.0.56:81/image/{camera_name}?w={width}&q={quality}" + url = f"http://10.0.0.226:8080/Umtxxf1uKMBniFblqeQ9KRbb6DDzN4/jpeg/GKlT2FfiSQ/{camera_name}/s.jpg" try: response = requests.get(url, stream=False, timeout=10.0) if response.ok: raw = response.content + logger.debug(f'Read {len(response.content)} byte image from HTTP server') tmp = np.frombuffer(raw, dtype="uint8") + logger.debug(f'Translated raw content into {tmp.shape} {type(tmp)} with element type {type(tmp[0])}.') jpg = cv2.imdecode(tmp, cv2.IMREAD_COLOR) + logger.debug(f'Decoded into {jpg.shape} jpeg {type(jpg)} with element type {type(jpg[0][0])}') hsv = cv2.cvtColor(jpg, cv2.COLOR_BGR2HSV) - (is_bad_image, _) = analyze_blue_iris_image(hsv) + logger.debug(f'Converted JPG into HSV {hsv.shape} HSV {type(hsv)} with element type {type(hsv[0][0])}') + (_, is_bad_image) = sanity_check_image(hsv) if not is_bad_image: logger.debug(f"Got a good image from {url}") return raw @@ -156,5 +168,5 @@ def fetch_camera_image( ) -> RawJpgHsv: try: return _fetch_camera_image(camera_name, width=width, quality=quality) - except decorator_utils.TimeoutError: + except exceptions.TimeoutError: return RawJpgHsv(None, None, None) diff --git a/dateparse/dateparse_utils.py b/dateparse/dateparse_utils.py index 026a513..21fdb83 100755 --- a/dateparse/dateparse_utils.py +++ b/dateparse/dateparse_utils.py @@ -1,5 +1,10 @@ #!/usr/bin/env python3 +""" +Parse dates in a variety of formats. + +""" + import datetime import functools import holidays # type: ignore @@ -249,6 +254,7 @@ class DateParser(dateparse_utilsListener): self.datetime: Optional[datetime.datetime] = None self.context: Dict[str, Any] = {} self.timedelta = datetime.timedelta(seconds=0) + self.saw_overt_year = False @staticmethod def _normalize_special_day_name(name: str) -> str: @@ -300,8 +306,10 @@ class DateParser(dateparse_utilsListener): next_last = self.context.get('special_next_last', '') if next_last == 'next': year += 1 + self.saw_overt_year = True elif next_last == 'last': year -= 1 + self.saw_overt_year = True # Holiday names if name == 'easte': @@ -360,6 +368,9 @@ class DateParser(dateparse_utilsListener): raise ParseException('Missing day') if 'year' not in self.context: self.context['year'] = self.today.year + self.saw_overt_year = False + else: + self.saw_overt_year = True # Handling "ides" and "nones" requires both the day and month. if ( @@ -610,11 +621,13 @@ class DateParser(dateparse_utilsListener): self.context['day'] = self.now_datetime.day self.context['month'] = self.now_datetime.month self.context['year'] = self.now_datetime.year + self.saw_overt_year = True elif txt[:4] == 'last': self.context['delta_int'] = -1 self.context['day'] = self.now_datetime.day self.context['month'] = self.now_datetime.month self.context['year'] = self.now_datetime.year + self.saw_overt_year = True else: raise ParseException(f'Bad next/last: {ctx.getText()}') @@ -843,6 +856,7 @@ class DateParser(dateparse_utilsListener): except Exception: raise ParseException(f'Bad year expression: {ctx.getText()}') else: + self.saw_overt_year = True self.context['year'] = year def exitSpecialDateMaybeYearExpr( diff --git a/executors.py b/executors.py index 92c5b34..cdbb811 100644 --- a/executors.py +++ b/executors.py @@ -745,10 +745,6 @@ class RemoteExecutor(BaseExecutor): xfer_latency = time.time() - start_ts logger.info(f"{bundle}: Copying done to {worker} in {xfer_latency:.1f}s.") except Exception as e: - logger.exception(e) - logger.error( - f'{bundle}: failed to send instructions to worker machine?!?' - ) assert bundle.worker is not None self.status.record_release_worker( bundle.worker, @@ -760,19 +756,30 @@ class RemoteExecutor(BaseExecutor): if is_original: # Weird. We tried to copy the code to the worker and it failed... # And we're the original bundle. We have to retry. + logger.exception(e) + logger.error( + f'{bundle}: Failed to send instructions to the worker machine?! ' + + 'This is not expected; we\'re the original bundle so this shouldn\'t ' + + 'be a race condition. Attempting an emergency retry...' + ) return self.emergency_retry_nasty_bundle(bundle) else: # This is actually expected; we're a backup. # There's a race condition where someone else # already finished the work and removed the source # code file before we could copy it. No biggie. + logger.warning( + f'{bundle}: Failed to send instructions to the worker machine... ' + + 'We\'re a backup and this may be caused by the original (or some ' + + 'other backup) already finishing this work. Ignoring this.' + ) return None # Kick off the work. Note that if this fails we let # wait_for_process deal with it. self.status.record_processing_began(uuid) cmd = (f'{SSH} {bundle.username}@{bundle.machine} ' - f'"source py39-venv/bin/activate &&' + f'"source py38-venv/bin/activate &&' f' /home/scott/lib/python_modules/remote_worker.py' f' --code_file {bundle.code_file} --result_file {bundle.result_file}"') logger.debug(f'{bundle}: Executing {cmd} in the background to kick off work...') @@ -889,7 +896,7 @@ class RemoteExecutor(BaseExecutor): if is_original: logger.debug(f"{bundle}: Unpickling {result_file}.") try: - with open(f'{result_file}', 'rb') as rb: + with open(result_file, 'rb') as rb: serialized = rb.read() result = cloudpickle.loads(serialized) except Exception as e: @@ -1112,30 +1119,10 @@ class DefaultExecutors(object): RemoteWorkerRecord( username = 'scott', machine = 'cheetah.house', - weight = 14, + weight = 25, count = 6, ), ) - if self.ping('video.house'): - logger.info('Found video.house') - pool.append( - RemoteWorkerRecord( - username = 'scott', - machine = 'video.house', - weight = 1, - count = 4, - ), - ) - if self.ping('gorilla.house'): - logger.info('Found gorilla.house') - pool.append( - RemoteWorkerRecord( - username = 'scott', - machine = 'gorilla.house', - weight = 2, - count = 4, - ), - ) if self.ping('meerkat.cabin'): logger.info('Found meerkat.cabin') pool.append( @@ -1146,16 +1133,16 @@ class DefaultExecutors(object): count = 2, ), ) - if self.ping('kiosk.house'): - logger.info('Found kiosk.house') - pool.append( - RemoteWorkerRecord( - username = 'pi', - machine = 'kiosk.house', - weight = 1, - count = 2, - ), - ) + # if self.ping('kiosk.house'): + # logger.info('Found kiosk.house') + # pool.append( + # RemoteWorkerRecord( + # username = 'pi', + # machine = 'kiosk.house', + # weight = 1, + # count = 2, + # ), + # ) if self.ping('hero.house'): logger.info('Found hero.house') pool.append( @@ -1172,18 +1159,18 @@ class DefaultExecutors(object): RemoteWorkerRecord( username = 'scott', machine = 'puma.cabin', - weight = 12, + weight = 25, count = 6, ), ) - if self.ping('puma.house'): - logger.info('Found puma.house') + if self.ping('backup.house'): + logger.info('Found backup.house') pool.append( RemoteWorkerRecord( username = 'scott', - machine = 'puma.house', - weight = 12, - count = 6, + machine = 'backup.house', + weight = 3, + count = 2, ), ) diff --git a/list_utils.py b/list_utils.py index 05512b5..992f1ae 100644 --- a/list_utils.py +++ b/list_utils.py @@ -2,7 +2,7 @@ from collections import Counter from itertools import islice -from typing import Any, Iterator, List, Mapping, Sequence +from typing import Any, Iterator, List, Mapping, Sequence, Tuple def shard(lst: List[Any], size: int) -> Iterator[Any]: @@ -200,6 +200,70 @@ def ngrams(lst: Sequence[Any], n): yield lst[i:i + n] +def permute(seq: Sequence[Any]): + """ + Returns all permutations of a sequence; takes O(N^2) time. + + >>> for x in permute('cat'): + ... print(x) + cat + cta + act + atc + tca + tac + + """ + yield from _permute(seq, "") + +def _permute(seq: Sequence[Any], path): + if len(seq) == 0: + yield path + + for i in range(len(seq)): + car = seq[i] + left = seq[0:i] + right = seq[i + 1:] + cdr = left + right + yield from _permute(cdr, path + car) + + +def binary_search(lst: Sequence[Any], target:Any) -> Tuple[bool, int]: + """Performs a binary search on lst (which must already be sorted). + Returns a Tuple composed of a bool which indicates whether the + target was found and an int which indicates the index closest to + target whether it was found or not. + + >>> a = [1, 4, 5, 6, 7, 9, 10, 11] + >>> binary_search(a, 4) + (True, 1) + + >>> binary_search(a, 12) + (False, 8) + + >>> binary_search(a, 3) + (False, 1) + + >>> binary_search(a, 2) + (False, 1) + + """ + return _binary_search(lst, target, 0, len(lst) - 1) + + +def _binary_search(lst: Sequence[Any], target: Any, low: int, high: int) -> Tuple[bool, int]: + if high >= low: + mid = (high + low) // 2 + if lst[mid] == target: + return (True, mid) + elif lst[mid] > target: + return _binary_search(lst, target, low, mid - 1) + else: + return _binary_search(lst, target, mid + 1, high) + else: + return (False, low) + + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/lockfile.py b/lockfile.py index b6a832e..ca190df 100644 --- a/lockfile.py +++ b/lockfile.py @@ -9,9 +9,20 @@ import signal import sys from typing import Optional +import config import decorator_utils +cfg = config.add_commandline_args( + f'Lockfile ({__file__})', + 'Args related to lockfiles') +cfg.add_argument( + '--lockfile_held_duration_warning_threshold_sec', + type=float, + default=10.0, + metavar='SECONDS', + help='If a lock is held for longer than this threshold we log a warning' +) logger = logging.getLogger(__name__) @@ -75,7 +86,7 @@ class LockFile(object): return True except OSError: pass - logger.debug(f'Failed; I could not acquire {self.lockfile}.') + logger.warning(f'Could not acquire {self.lockfile}.') return False def acquire_with_retries( @@ -108,12 +119,19 @@ class LockFile(object): def __enter__(self): if self.acquire_with_retries(): + self.locktime = datetime.datetime.now().timestamp() return self msg = f"Couldn't acquire {self.lockfile}; giving up." logger.warning(msg) raise LockFileException(msg) def __exit__(self, type, value, traceback): + if self.locktime: + ts = datetime.datetime.now().timestamp() + duration = ts - self.locktime + if duration >= config.config['lockfile_held_duration_warning_threshold_sec']: + str_duration = datetime_utils.describe_duration_briefly(duration) + logger.warning(f'Held {self.lockfile} for {str_duration}') self.release() def __del__(self): @@ -151,15 +169,16 @@ class LockFile(object): try: os.kill(contents.pid, 0) except OSError: - logger.debug('The pid seems stale; killing the lock.') + logger.warning(f'Lockfile {self.lockfile}\'s pid ({contents.pid}) is stale; ' + + 'force acquiring') self.release() # Has the lock expiration expired? if contents.expiration_timestamp is not None: now = datetime.datetime.now().timestamp() if now > contents.expiration_datetime: - logger.debug('The expiration time has passed; ' + - 'killing the lock') + logger.warning(f'Lockfile {self.lockfile} expiration time has passed; ' + + 'force acquiring') self.release() except Exception: pass diff --git a/logging_utils.py b/logging_utils.py index 819e3d3..278cbf0 100644 --- a/logging_utils.py +++ b/logging_utils.py @@ -43,8 +43,8 @@ cfg.add_argument( cfg.add_argument( '--logging_format', type=str, - default='%(levelname).1s:%(asctime)s: %(message)s', - help='The format for lines logged via the logger module.' + default=None, + help='The format for lines logged via the logger module. See: https://docs.python.org/3/library/logging.html#formatter-objects' ) cfg.add_argument( '--logging_date_format', @@ -86,6 +86,16 @@ cfg.add_argument( default=False, help='Should we log to localhost\'s syslog.' ) +cfg.add_argument( + '--logging_syslog_facility', + type=str, + default = 'USER', + choices=['NOTSET', 'AUTH', 'AUTH_PRIV', 'CRON', 'DAEMON', 'FTP', 'KERN', 'LPR', 'MAIL', 'NEWS', + 'SYSLOG', 'USER', 'UUCP', 'LOCAL0', 'LOCAL1', 'LOCAL2', 'LOCAL3', 'LOCAL4', 'LOCAL5', + 'LOCAL6', 'LOCAL7'], + metavar='SYSLOG_FACILITY_LIST', + help='The default syslog message facility identifier', +) cfg.add_argument( '--logging_debug_threads', action=argparse_utils.ActionNoYes, @@ -382,7 +392,14 @@ def initialize_logging(logger=None) -> logging.Logger: if not isinstance(default_logging_level, int): raise ValueError('Invalid level: %s' % config.config['logging_level']) - fmt = config.config['logging_format'] + if config.config['logging_format']: + fmt = config.config['logging_format'] + else: + if config.config['logging_syslog']: + fmt = '%(levelname).1s:%(filename)s[%(process)d]: %(message)s' + else: + fmt = '%(levelname).1s:%(asctime)s: %(message)s' + if config.config['logging_debug_threads']: fmt = f'%(process)d.%(thread)d|{fmt}' if config.config['logging_debug_modules']: @@ -390,7 +407,10 @@ def initialize_logging(logger=None) -> logging.Logger: if config.config['logging_syslog']: if sys.platform not in ('win32', 'cygwin'): - handler = SysLogHandler() + if config.config['logging_syslog_facility']: + facility_name = 'LOG_' + config.config['logging_syslog_facility'] + facility = SysLogHandler.__dict__.get(facility_name, SysLogHandler.LOG_USER) + handler = SysLogHandler(facility=SysLogHandler.LOG_CRON, address='/dev/log') handler.setFormatter( MillisecondAwareFormatter( fmt=fmt, diff --git a/ml/model_trainer.py b/ml/model_trainer.py index f61b8e7..acd7218 100644 --- a/ml/model_trainer.py +++ b/ml/model_trainer.py @@ -246,12 +246,12 @@ class TrainingBlueprint(ABC): y.pop() if self.spec.delete_bad_inputs: - msg = f"WARNING: {filename}: missing features or label. DELETING." + msg = f"WARNING: {filename}: missing features or label; expected {self.spec.feature_count} but saw {len(x)}. DELETING." print(msg, file=sys.stderr) logger.warning(msg) os.remove(filename) else: - msg = f"WARNING: {filename}: missing features or label. Skipped." + msg = f"WARNING: {filename}: missing features or label; expected {self.spec.feature_count} but saw {len(x)}. Skipping." print(msg, file=sys.stderr) logger.warning(msg) return (X, y) diff --git a/pip_install.sh b/pip_install.sh index 832ae9c..1ee08c9 100755 --- a/pip_install.sh +++ b/pip_install.sh @@ -1,5 +1,7 @@ #!/bin/bash +# Install a bunch of pip modules that scott library depends upon. + set -e python3 -m ensurepip --upgrade diff --git a/remote_worker.py b/remote_worker.py index c04ac65..0086c40 100755 --- a/remote_worker.py +++ b/remote_worker.py @@ -83,6 +83,7 @@ def main() -> None: in_file = config.config['code_file'] out_file = config.config['result_file'] + stop_thread = None if config.config['watch_for_cancel']: (thread, stop_thread) = watch_for_cancel() @@ -130,8 +131,9 @@ def main() -> None: stop_thread.set() sys.exit(-1) - stop_thread.set() - thread.join() + if stop_thread is not None: + stop_thread.set() + thread.join() if __name__ == '__main__': diff --git a/site_config.py b/site_config.py index 4968523..caaf3d8 100644 --- a/site_config.py +++ b/site_config.py @@ -33,7 +33,7 @@ class SiteConfig(object): network_netmask: str network_router_ip: str presence_location: Location - is_anyone_present: Callable[None, bool] + is_anyone_present: Callable arper_minimum_device_count: int @@ -53,7 +53,7 @@ def get_location(): """ Returns location as an enum instead of a string. - >>> from locations import Location + >>> from type.locations import Location >>> location = get_location() >>> location == Location.HOUSE or location == Location.CABIN True diff --git a/smart_home/cameras.py b/smart_home/cameras.py index 8137012..51a95e9 100644 --- a/smart_home/cameras.py +++ b/smart_home/cameras.py @@ -16,6 +16,7 @@ class BaseCamera(dev.Device): 'outside_driveway_camera': 'driveway', 'outside_doorbell_camera': 'doorbell', 'outside_front_door_camera': 'front_door', + 'crawlspace_camera': 'crawlspace', } def __init__(self, name: str, mac: str, keywords: str = "") -> None: @@ -23,5 +24,9 @@ class BaseCamera(dev.Device): self.camera_name = BaseCamera.camera_mapping.get(name, None) def get_stream_url(self) -> str: - assert self.camera_name is not None - return f'http://10.0.0.226:8080/Umtxxf1uKMBniFblqeQ9KRbb6DDzN4/mp4/GKlT2FfiSQ/{self.camera_name}/s.mp4' + name = self.camera_name + assert name is not None + if name == 'driveway': + return f'http://10.0.0.226:8080/Umtxxf1uKMBniFblqeQ9KRbb6DDzN4/mjpeg/GKlT2FfiSQ/driveway' + else: + return f'http://10.0.0.226:8080/Umtxxf1uKMBniFblqeQ9KRbb6DDzN4/mp4/GKlT2FfiSQ/{name}/s.mp4' diff --git a/smart_home/lights.py b/smart_home/lights.py index 1c4081c..e23569a 100644 --- a/smart_home/lights.py +++ b/smart_home/lights.py @@ -318,7 +318,10 @@ class TPLinkLight(BaseLight): @overrides def is_on(self) -> bool: - return self.get_on_duration_seconds() > 0 + self.info = self.get_info() + if self.info is None: + raise Exception('Unable to get info?') + return self.info.get("relay_state", "0") == "1" @overrides def is_off(self) -> bool: diff --git a/smart_home/outlets.py b/smart_home/outlets.py index f34d574..68dfd2b 100644 --- a/smart_home/outlets.py +++ b/smart_home/outlets.py @@ -3,6 +3,8 @@ """Utilities for dealing with the smart outlets.""" from abc import abstractmethod +import asyncio +import atexit import datetime import json import logging @@ -10,11 +12,16 @@ import os import re import subprocess import sys -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional + +from meross_iot.http_api import MerossHttpClient +from meross_iot.manager import MerossManager import argparse_utils import config +import decorator_utils import logging_utils +import scott_secrets import smart_home.device as dev from google_assistant import ask_google, GoogleResponse from decorator_utils import timeout, memoized @@ -227,3 +234,81 @@ class GoogleOutlet(BaseOutlet): def is_off(self) -> bool: return not self.is_on() + + +@decorator_utils.singleton +class MerossWrapper(object): + """Note that instantiating this class causes HTTP traffic with an + external Meross server. Meross blocks customers who hit their + servers too aggressively so MerossOutlet is lazy about creating + instances of this class. + + """ + + def __init__(self): + self.loop = asyncio.get_event_loop() + self.email = os.environ.get('MEROSS_EMAIL') or scott_secrets.MEROSS_EMAIL + self.password = os.environ.get('MEROSS_PASSWORD') or scott_secrets.MEROSS_PASSWORD + self.devices = self.loop.run_until_complete(self.find_meross_devices()) + atexit.register(self.loop.close) + + async def find_meross_devices(self) -> List[Any]: + http_api_client = await MerossHttpClient.async_from_user_password( + email=self.email, password=self.password + ) + + # Setup and start the device manager + manager = MerossManager(http_client=http_api_client) + await manager.async_init() + + # Discover devices + await manager.async_device_discovery() + devices = manager.find_devices() + for device in devices: + await device.async_update() + return devices + + def get_meross_device_by_name(self, name: str) -> Optional[Any]: + name = name.lower() + name = name.replace('_', ' ') + for device in self.devices: + if device.name.lower() == name: + return device + return None + + +class MerossOutlet(BaseOutlet): + def __init__(self, name: str, mac: str, keywords: str = '') -> None: + super().__init__(name, mac, keywords) + self.meross_wrapper = None + self.device = None + + def lazy_initialize_device(self): + """If we make too many calls to Meross they will block us; only talk + to them when someone actually wants to control a device.""" + if self.meross_wrapper is None: + self.meross_wrapper = MerossWrapper() + self.device = self.meross_wrapper.get_meross_device_by_name(self.name) + if self.device is None: + raise Exception(f'{self.name} is not a known Meross device?!') + + def turn_on(self) -> bool: + self.lazy_initialize_device() + self.meross_wrapper.loop.run_until_complete( + self.device.async_turn_on() + ) + return True + + def turn_off(self) -> bool: + self.lazy_initialize_device() + self.meross_wrapper.loop.run_until_complete( + self.device.async_turn_off() + ) + return True + + def is_on(self) -> bool: + self.lazy_initialize_device() + return self.device.is_on() + + def is_off(self) -> bool: + return not self.is_on() diff --git a/smart_home/registry.py b/smart_home/registry.py index ae57a73..23584e1 100644 --- a/smart_home/registry.py +++ b/smart_home/registry.py @@ -165,6 +165,9 @@ class SmartHomeRegistry(object): else: logger.debug(' ...a TPLinkOutlet') return outlets.TPLinkOutlet(name, mac, kws) + elif 'meross' in kws.lower(): + logger.debug(' ...a MerossOutlet') + return outlets.MerossOutlet(name, mac, kws) elif 'goog' in kws.lower(): logger.debug(' ...a GoogleOutlet') return outlets.GoogleOutlet(name, mac, kws) diff --git a/smart_home/thermometers.py b/smart_home/thermometers.py new file mode 100644 index 0000000..fe5eed1 --- /dev/null +++ b/smart_home/thermometers.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import logging +from typing import Optional +import urllib.request + +logger = logging.getLogger() + + +class ThermometerRegistry(object): + def __init__(self): + self.thermometers = { + 'house_outside': ('10.0.0.75', 'outside_temp'), + 'house_inside_downstairs': ('10.0.0.75', 'inside_downstairs_temp'), + 'house_inside_upstairs': ('10.0.0.75', 'inside_upstairs_temp'), + 'house_computer_closet': ('10.0.0.75', 'computer_closet_temp'), + 'house_crawlspace': ('10.0.0.75', 'crawlspace_temp'), + 'cabin_outside': ('192.168.0.107', 'outside_temp'), + 'cabin_inside': ('192.168.0.107', 'inside_temp'), + 'cabin_crawlspace': ('192.168.0.107', 'crawlspace_temp'), + 'cabin_hottub': ('192.168.0.107', 'hottub_temp'), + } + + def read_temperature( + self, location: str, *, convert_to_fahrenheit=False + ) -> Optional[float]: + record = self.thermometers.get(location, None) + if record is None: + logger.error( + f'Location {location} is not known. Valid locations are {self.thermometers.keys()}.' + ) + return None + url = f'http://{record[0]}/~pi/{record[1]}' + logger.debug(f'Constructed URL: {url}') + try: + www = urllib.request.urlopen(url, timeout=3) + temp = www.read().decode('utf-8') + temp = float(temp) + if convert_to_fahrenheit: + temp *= (9/5) + temp += 32.0 + temp = round(temp) + except Exception as e: + logger.exception(e) + logger.error(f'Failed to read temperature at URL: {url}') + temp = None + finally: + if www is not None: + www.close() + return temp diff --git a/waitable_presence.py b/waitable_presence.py new file mode 100644 index 0000000..9e0a9d0 --- /dev/null +++ b/waitable_presence.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +"""A PresenceDetector that is waitable. This is not part of +base_presence.py because I do not want to bring these dependencies +into that lower-level module (especially state_tracker). + +""" + +import datetime +import logging +from typing import Optional, Tuple + +from overrides import overrides + +import base_presence +from type.locations import Location +import site_config +import state_tracker + +logger = logging.getLogger(__name__) + + +class WaitablePresenceDetectorWithMemory(state_tracker.WaitableAutomaticStateTracker): + """ + This is a waitable class that keeps a PresenceDetector internally + and periodically polls it to detect changes in presence in a + particular location. Example suggested usage pattern: + + detector = waitable_presence.WaitablePresenceDetectorWithMemory(60.0) + while True: + changed = detector.wait(timeout=60 * 5) # or, None for "forever" + (someone_is_home, since) = detector.is_someone_home() + if changed: + detector.reset() + logger.debug( + f'someone_is_home={someone_is_home}, since={since}, changed={changed}' + ) + """ + + def __init__( + self, + override_update_interval_sec: float = 60.0, + override_location: Location = site_config.get_location(), + ) -> None: + self.last_someone_is_home: Optional[bool] = None + self.someone_is_home: Optional[bool] = None + self.everyone_gone_since: Optional[datetime.datetime] = None + self.someone_home_since: Optional[datetime.datetime] = None + self.location = override_location + self.detector: base_presence.PresenceDetection = base_presence.PresenceDetection() + super().__init__( + { + 'poll_presence': override_update_interval_sec, + 'check_detector': override_update_interval_sec * 5, + } + ) + + @overrides + def update( + self, + update_id: str, + now: datetime.datetime, + last_invocation: Optional[datetime.datetime], + ) -> None: + if update_id == 'poll_presence': + self.poll_presence(now) + elif update_id == 'check_detector': + self.check_detector() + else: + raise Exception(f'Unknown update type {update_id} in {__file__}') + + def poll_presence(self, now: datetime.datetime) -> None: + logger.debug(f'Checking presence in {self.location} now...') + self.detector.update() + if self.detector.is_anyone_in_location_now(self.location): + self.someone_is_home = True + self.someone_home_since = now + else: + self.someone_is_home = False + self.everyone_gone_since = now + if self.someone_is_home != self.last_someone_is_home: + self.something_changed() + self.last_someone_is_home = self.someone_is_home + + def check_detector(self) -> None: + if len(self.detector.dark_locations) > 0: + logger.debug('PresenceDetector is incomplete; trying to reinitialize...') + self.detector = base_presence.PresenceDetection() + + def is_someone_home(self) -> Tuple[bool, datetime.datetime]: + """Returns a tuple of a bool that indicates whether someone is home + and a datetime that indicates how long either someone has been + home or no one has been home. + + """ + if self.someone_is_home is None: + raise Exception("Too Soon!") + if self.someone_is_home: + return (True, self.someone_home_since) + else: + return (False, self.everyone_gone_since)