From 1574e8a3a8982fab9278ad534f9427d464e4bffb Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Fri, 16 Jul 2021 21:43:44 -0700 Subject: [PATCH 1/1] Various --- bootstrap.py | 21 ++++-- config.py | 16 ++-- dateparse/dateparse_utils.py | 9 ++- directory_filter.py | 3 +- google_assistant.py | 6 ++ logging_utils.py | 138 ++++++++++++++++++++++++----------- stopwatch.py | 1 - text_utils.py | 25 +++++-- unittest_utils.py | 12 ++- 9 files changed, 159 insertions(+), 72 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index 3c886ef..3b03b3a 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -12,7 +12,7 @@ import traceback from argparse_utils import ActionNoYes import config - +import logging_utils logger = logging.getLogger(__name__) @@ -48,9 +48,14 @@ def initialize(entry_point): @functools.wraps(entry_point) def initialize_wrapper(*args, **kwargs): sys.excepthook = handle_uncaught_exception - config.parse(entry_point.__globals__['__file__']) + if ( + '__globals__' in entry_point.__dict__ and + '__file__' in entry_point.__globals__ + ): + config.parse(entry_point.__globals__['__file__']) + else: + config.parse(None) - import logging_utils logging_utils.initialize_logging(logging.getLogger()) config.late_logging() @@ -58,8 +63,8 @@ def initialize(entry_point): logger.debug(f'Starting {entry_point.__name__} (program entry point)') ret = None - import timer - with timer.Timer() as t: + import stopwatch + with stopwatch.Timer() as t: ret = entry_point(*args, **kwargs) logger.debug( f'{entry_point.__name__} (program entry point) returned {ret}.' @@ -67,13 +72,13 @@ def initialize(entry_point): walltime = t() (utime, stime, cutime, cstime, elapsed_time) = os.times() - logger.debug(f'\n' + logger.debug('\n' f'user: {utime}s\n' f'system: {stime}s\n' f'child user: {cutime}s\n' f'child system: {cstime}s\n' - f'elapsed: {elapsed_time}s\n' - f'walltime: {walltime}s\n') + f'machine uptime: {elapsed_time}s\n' + f'walltime: {walltime}s') if ret != 0: logger.info(f'Exit {ret}') else: diff --git a/config.py b/config.py index dccfc27..81351cf 100644 --- a/config.py +++ b/config.py @@ -69,7 +69,7 @@ import os import pprint import re import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional # This module is commonly used by others in here and should avoid # taking any unnecessary dependencies back on them. @@ -157,7 +157,7 @@ def is_flag_already_in_argv(var: str): return False -def parse(entry_module: str) -> Dict[str, Any]: +def parse(entry_module: Optional[str]) -> Dict[str, Any]: """Main program should call this early in main()""" global config_parse_called if config_parse_called: @@ -166,17 +166,19 @@ def parse(entry_module: str) -> Dict[str, Any]: global saved_messages # If we're about to do the usage message dump, put the main module's - # argument group first in the list, please. + # argument group first in the list (if possible), please. reordered_action_groups = [] prog = sys.argv[0] + for arg in sys.argv: if arg == '--help' or arg == '-h': - print(entry_module) for group in args._action_groups: - if entry_module in group.title or prog in group.title: - reordered_action_groups.insert(0, group) + if entry_module is not None and entry_module in group.title: + reordered_action_groups.insert(0, group) # prepend + elif prog in group.title: + reordered_action_groups.insert(0, group) # prepend else: - reordered_action_groups.append(group) + reordered_action_groups.append(group) # append args._action_groups = reordered_action_groups # Examine the environment variables to settings that match diff --git a/dateparse/dateparse_utils.py b/dateparse/dateparse_utils.py index 05bee8d..4decb81 100755 --- a/dateparse/dateparse_utils.py +++ b/dateparse/dateparse_utils.py @@ -21,7 +21,7 @@ from datetime_utils import ( from dateparse.dateparse_utilsLexer import dateparse_utilsLexer # type: ignore from dateparse.dateparse_utilsListener import dateparse_utilsListener # type: ignore from dateparse.dateparse_utilsParser import dateparse_utilsParser # type: ignore -from decorator_utils import decorate_matching_methods_with +import decorator_utils logger = logging.getLogger(__name__) @@ -80,14 +80,17 @@ class RaisingErrorListener(antlr4.DiagnosticErrorListener): pass -@decorate_matching_methods_with( +@decorator_utils.decorate_matching_methods_with( debug_parse, acl=acl.StringWildcardBasedACL( allowed_patterns=[ 'enter*', 'exit*', ], - denied_patterns=None, + denied_patterns=[ + 'enterEveryRule', + 'exitEveryRule' + ], order_to_check_allow_deny=acl.Order.DENY_ALLOW, default_answer=False ) diff --git a/directory_filter.py b/directory_filter.py index d275cf2..5504609 100644 --- a/directory_filter.py +++ b/directory_filter.py @@ -10,10 +10,9 @@ class DirectoryFileFilter(object): content to-be-written is identical to the contents of the file; skip the write. """ - def __init__(self, directory: str): - import file_utils super().__init__() + import file_utils if not file_utils.does_directory_exist(directory): raise ValueError(directory) self.directory = directory diff --git a/google_assistant.py b/google_assistant.py index 500a909..71301e4 100644 --- a/google_assistant.py +++ b/google_assistant.py @@ -49,6 +49,7 @@ def tell_google(cmd: str, *, recognize_speech=True) -> GoogleResponse: def ask_google(cmd: str, *, recognize_speech=True) -> GoogleResponse: + logging.debug(f"Asking google: '{cmd}'") payload = { "command": cmd, "user": config.config['google_assistant_username'], @@ -63,6 +64,10 @@ def ask_google(cmd: str, *, recognize_speech=True) -> GoogleResponse: j = r.json() success = bool(j["success"]) response = j["response"] if success else j["error"] + if success: + logger.debug('Google request succeeded.') + if len(response) > 0: + logger.debug(f"Google said: '{response}'") audio = f"{config.config['google_assistant_bridge']}{j['audio']}" if recognize_speech: recognizer = sr.Recognizer() @@ -77,6 +82,7 @@ def ask_google(cmd: str, *, recognize_speech=True) -> GoogleResponse: audio_transcription = recognizer.recognize_google( speech, ) + logger.debug(f"Transcription: '{audio_transcription}'") else: logger.error( f'HTTP request to {url} with {payload} failed; code {r.status_code}' diff --git a/logging_utils.py b/logging_utils.py index 328ea6f..b7fd11f 100644 --- a/logging_utils.py +++ b/logging_utils.py @@ -4,6 +4,7 @@ import contextlib import datetime +import enum import logging from logging.handlers import RotatingFileHandler, SysLogHandler import os @@ -92,6 +93,16 @@ cfg.add_argument( help='logging.info also prints to stdout.' ) +# See also: OutputMultiplexer/OutputContext +cfg.add_argument( + '--logging_captures_prints', + action=argparse_utils.ActionNoYes, + default=False, + help='When calling print also log.info too' +) + +built_in_print = print + class OnlyInfoFilter(logging.Filter): def filter(self, record): @@ -136,12 +147,7 @@ def initialize_logging(logger=None) -> logging.Logger: fmt = f'%(process)d.%(thread)d|{fmt}' if config.config['logging_syslog']: - if sys.platform in ('win32', 'cygwin'): - print( - "WARNING: Current platform does not support syslog; IGNORING.", - file=sys.stderr - ) - else: + if sys.platform not in ('win32', 'cygwin'): handler = SysLogHandler() # for k, v in encoded_priorities.items(): # handler.encodePriority(k, v) @@ -154,7 +160,7 @@ def initialize_logging(logger=None) -> logging.Logger: handler.setLevel(numeric_level) handlers.append(handler) - if config.config['logging_filename'] is not None: + if config.config['logging_filename']: handler = RotatingFileHandler( config.config['logging_filename'], maxBytes = config.config['logging_filename_maxsize'], @@ -185,12 +191,28 @@ def initialize_logging(logger=None) -> logging.Logger: for handler in handlers: logger.addHandler(handler) + if config.config['logging_info_is_print']: handler = logging.StreamHandler(sys.stdout) handler.addFilter(OnlyInfoFilter()) logger.addHandler(handler) + logger.setLevel(numeric_level) logger.propagate = False + + if config.config['logging_captures_prints']: + import builtins + global built_in_print + + def print_and_also_log(*arg, **kwarg): + f = kwarg.get('file', None) + if f == sys.stderr: + logger.warning(*arg) + else: + logger.info(*arg) + built_in_print(*arg, **kwarg) + builtins.print = print_and_also_log + return logger @@ -212,30 +234,32 @@ def dprint(*args, **kwargs) -> None: print(*args, file=sys.stderr, **kwargs) -class OutputSink(object): - - # Bits in the destination_bitv bitvector. Used to indicate the - # output destination. - STDOUT = 0x1 - STDERR = 0x2 - LOG_DEBUG = 0x4 # -\ - LOG_INFO = 0x8 # | - LOG_WARNING = 0x10 # > Should provide logger to the c'tor. - LOG_ERROR = 0x20 # | - LOG_CRITICAL = 0x40 # _/ - FILENAME = 0x80 # Must provide a filename to the c'tor. - HLOG = 0x100 - - ALL_LOG_DESTINATIONS = ( - LOG_DEBUG | LOG_INFO | LOG_WARNING | LOG_ERROR | LOG_CRITICAL - ) - ALL_OUTPUT_DESTINATIONS = 0x1FF +class OutputMultiplexer(object): + + class Destination(enum.IntEnum): + """Bits in the destination_bitv bitvector. Used to indicate the + output destination.""" + STDOUT = 0x1 + STDERR = 0x2 + LOG_DEBUG = 0x4 # -\ + LOG_INFO = 0x8 # | + LOG_WARNING = 0x10 # > Should provide logger to the c'tor. + LOG_ERROR = 0x20 # | + LOG_CRITICAL = 0x40 # _/ + FILENAME = 0x80 # Must provide a filename to the c'tor. + FILEHANDLE = 0x100 # Must provide a handle to the c'tor. + HLOG = 0x200 + ALL_LOG_DESTINATIONS = ( + LOG_DEBUG | LOG_INFO | LOG_WARNING | LOG_ERROR | LOG_CRITICAL + ) + ALL_OUTPUT_DESTINATIONS = 0x2FF def __init__(self, destination_bitv: int, *, logger=None, - filename=None): + filename=None, + handle=None): if logger is None: logger = logging.getLogger(None) self.logger = logger @@ -243,21 +267,35 @@ class OutputSink(object): if filename is not None: self.f = open(filename, "wb", buffering=0) else: - if self.destination_bitv & OutputSink.FILENAME: + if self.destination_bitv & OutputMultiplexer.FILENAME: raise ValueError( "Filename argument is required if bitv & FILENAME" ) self.f = None + + if handle is not None: + self.h = handle + else: + if self.destination_bitv & OutputMultiplexer.FILEHANDLE: + raise ValueError( + "Handle argument is required if bitv & FILEHANDLE" + ) + self.h = None + self.set_destination_bitv(destination_bitv) def get_destination_bitv(self): return self.destination_bitv def set_destination_bitv(self, destination_bitv: int): - if destination_bitv & self.FILENAME and self.f is None: + if destination_bitv & self.Destination.FILENAME and self.f is None: raise ValueError( "Filename argument is required if bitv & FILENAME" ) + if destination_bitv & self.Destination.FILEHANDLE and self.h is None: + raise ValueError( + "Handle argument is required if bitv & FILEHANDLE" + ) self.destination_bitv = destination_bitv def print(self, *args, **kwargs): @@ -277,28 +315,39 @@ class OutputSink(object): sep = " " if end is None: end = "\n" - if self.destination_bitv & self.STDOUT: + if self.destination_bitv & self.Destination.STDOUT: print(buf, file=sys.stdout, sep=sep, end=end) - if self.destination_bitv & self.STDERR: + if self.destination_bitv & self.Destination.STDERR: print(buf, file=sys.stderr, sep=sep, end=end) if end == '\n': buf += '\n' - if self.destination_bitv & self.FILENAME and self.f is not None: + if ( + self.destination_bitv & self.Destination.FILENAME and + self.f is not None + ): self.f.write(buf.encode('utf-8')) self.f.flush() + + if ( + self.destination_bitv & self.Destination.FILEHANDLE and + self.h is not None + ): + self.h.write(buf) + self.h.flush() + buf = strip_escape_sequences(buf) if self.logger is not None: - if self.destination_bitv & self.LOG_DEBUG: + if self.destination_bitv & self.Destination.LOG_DEBUG: self.logger.debug(buf) - if self.destination_bitv & self.LOG_INFO: + if self.destination_bitv & self.Destination.LOG_INFO: self.logger.info(buf) - if self.destination_bitv & self.LOG_WARNING: + if self.destination_bitv & self.Destination.LOG_WARNING: self.logger.warning(buf) - if self.destination_bitv & self.LOG_ERROR: + if self.destination_bitv & self.Destination.LOG_ERROR: self.logger.error(buf) - if self.destination_bitv & self.LOG_CRITICAL: + if self.destination_bitv & self.Destination.LOG_CRITICAL: self.logger.critical(buf) - if self.destination_bitv & self.HLOG: + if self.destination_bitv & self.Destination.HLOG: hlog(buf) def close(self): @@ -306,18 +355,23 @@ class OutputSink(object): self.f.close() -class OutputContext(OutputSink, contextlib.ContextDecorator): +class OutputContext(OutputMultiplexer, contextlib.ContextDecorator): def __init__(self, - destination_bitv: int, + destination_bitv: OutputMultiplexer.Destination, *, logger=None, - filename=None): - super().__init__(destination_bitv, logger=logger, filename=filename) + filename=None, + handle=None): + super().__init__( + destination_bitv, + logger=logger, + filename=filename, + handle=handle) def __enter__(self): return self - def __exit__(self, etype, value, traceback): + def __exit__(self, etype, value, traceback) -> bool: super().close() if etype is not None: return False diff --git a/stopwatch.py b/stopwatch.py index 752c7ed..d54af87 100644 --- a/stopwatch.py +++ b/stopwatch.py @@ -16,7 +16,6 @@ class Timer(object): def __init__(self) -> None: self.start = None self.end = None - pass def __enter__(self) -> Callable[[], float]: self.start = time.perf_counter() diff --git a/text_utils.py b/text_utils.py index 93e4b63..1a8fa18 100644 --- a/text_utils.py +++ b/text_utils.py @@ -5,7 +5,7 @@ from collections import defaultdict import math import sys -from typing import List, NamedTuple +from typing import List, NamedTuple, Optional from ansi import fg, reset @@ -171,15 +171,24 @@ def generate_padded_columns(text: List[str]) -> str: class Indenter: """ - with Indenter() as i: - i.print('test') - with i: - i.print('-ing') + with Indenter(pad_count = 8) as i: + i.print('test') with i: - i.print('1, 2, 3') + i.print('-ing') + with i: + i.print('1, 2, 3') """ - def __init__(self): + def __init__(self, + *, + pad_prefix: Optional[str] = None, + pad_char: str = ' ', + pad_count: int = 4): self.level = -1 + if pad_prefix is not None: + self.pad_prefix = pad_prefix + else: + self.pad_prefix = '' + self.padding = pad_char * pad_count def __enter__(self): self.level += 1 @@ -193,4 +202,4 @@ class Indenter: def print(self, *arg, **kwargs): import string_utils text = string_utils.sprintf(*arg, **kwargs) - print(" " * self.level + text) + print(self.pad_prefix + self.padding * self.level + text, end='') diff --git a/unittest_utils.py b/unittest_utils.py index 99ac81d..e7090bc 100644 --- a/unittest_utils.py +++ b/unittest_utils.py @@ -8,12 +8,15 @@ etc... this works fine but it's a little hacky so caveat emptor. import functools import inspect +import io import logging import pickle import random import statistics +import sys import time -from typing import Callable +import tempfile +from typing import Callable, Iterable import unittest import bootstrap @@ -142,3 +145,10 @@ def check_all_methods_for_perf_regressions(prefix='test_'): logger.debug(f'Wrapping {cls.__name__}:{name}.') return cls return decorate_the_testcase + + +def breakpoint(): + import pdb + pdb.set_trace() + + -- 2.45.2