X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=unittest_utils.py;h=81b339ae3485d05e6f76248aa8fbeb70f52ef2a3;hb=5317c50ce7a96a37acfab3800c0935580766dbbf;hp=d63f2b566b51b33bc794340b0d425d29a7e89142;hpb=559c60c169223c7c6833e9beedf978a8ffdd3926;p=python_utils.git diff --git a/unittest_utils.py b/unittest_utils.py index d63f2b5..81b339a 100644 --- a/unittest_utils.py +++ b/unittest_utils.py @@ -18,7 +18,7 @@ import random import statistics import time import tempfile -from typing import Callable, Dict, List +from typing import Any, Callable, Dict, List, Optional import unittest import warnings @@ -32,8 +32,8 @@ import sqlalchemy as sa logger = logging.getLogger(__name__) cfg = config.add_commandline_args( - f'Logging ({__file__})', - 'Args related to function decorators') + f'Logging ({__file__})', 'Args related to function decorators' +) cfg.add_argument( '--unittests_ignore_perf', action='store_true', @@ -44,34 +44,34 @@ cfg.add_argument( '--unittests_num_perf_samples', type=int, default=50, - help='The count of perf timing samples we need to see before blocking slow runs on perf grounds' + help='The count of perf timing samples we need to see before blocking slow runs on perf grounds', ) cfg.add_argument( '--unittests_drop_perf_traces', type=str, nargs=1, default=None, - help='The identifier (i.e. file!test_fixture) for which we should drop all perf data' + help='The identifier (i.e. file!test_fixture) for which we should drop all perf data', ) cfg.add_argument( '--unittests_persistance_strategy', choices=['FILE', 'DATABASE'], default='DATABASE', - help='Should we persist perf data in a file or db?' + help='Should we persist perf data in a file or db?', ) cfg.add_argument( '--unittests_perfdb_filename', type=str, metavar='FILENAME', default=f'{os.environ["HOME"]}/.python_unittest_performance_db', - help='File in which to store perf data (iff --unittests_persistance_strategy is FILE)' + help='File in which to store perf data (iff --unittests_persistance_strategy is FILE)', ) cfg.add_argument( '--unittests_perfdb_spec', type=str, metavar='DBSPEC', default='mariadb+pymysql://python_unittest:@db.house:3306/python_unittest_performance', - help='Db connection spec for perf data (iff --unittest_persistance_strategy is DATABASE)' + help='Db connection spec for perf data (iff --unittest_persistance_strategy is DATABASE)', ) # >>> This is the hacky business, FYI. <<< @@ -83,7 +83,7 @@ class PerfRegressionDataPersister(ABC): pass @abstractmethod - def load_performance_data(self) -> Dict[str, List[float]]: + def load_performance_data(self, method_id: str) -> Dict[str, List[float]]: pass @abstractmethod @@ -98,7 +98,7 @@ class PerfRegressionDataPersister(ABC): class FileBasedPerfRegressionDataPersister(PerfRegressionDataPersister): def __init__(self, filename: str): self.filename = filename - self.traces_to_delete = [] + self.traces_to_delete: List[str] = [] def load_performance_data(self, method_id: str) -> Dict[str, List[float]]: with open(self.filename, 'rb') as f: @@ -128,7 +128,7 @@ class DatabasePerfRegressionDataPersister(PerfRegressionDataPersister): f'SELECT * FROM runtimes_by_function WHERE function = "{method_id}";' ) ) - ret = {method_id: []} + ret: Dict[str, List[float]] = {method_id: []} for result in results.all(): ret[method_id].append(result['runtime']) results.close() @@ -155,19 +155,23 @@ def check_method_for_perf_regressions(func: Callable) -> Callable: message if it has become too slow. """ + @functools.wraps(func) def wrapper_perf_monitor(*args, **kwargs): + if config.config['unittests_ignore_perf']: + return func(*args, **kwargs) + if config.config['unittests_persistance_strategy'] == 'FILE': filename = config.config['unittests_perfdb_filename'] helper = FileBasedPerfRegressionDataPersister(filename) elif config.config['unittests_persistance_strategy'] == 'DATABASE': dbspec = config.config['unittests_perfdb_spec'] - dbspec = dbspec.replace('', scott_secrets.MARIADB_UNITTEST_PERF_PASSWORD) + dbspec = dbspec.replace( + '', scott_secrets.MARIADB_UNITTEST_PERF_PASSWORD + ) helper = DatabasePerfRegressionDataPersister(dbspec) else: - raise Exception( - 'Unknown/unexpected --unittests_persistance_strategy value' - ) + raise Exception('Unknown/unexpected --unittests_persistance_strategy value') func_id = function_utils.function_identifier(func) func_name = func.__name__ @@ -198,25 +202,16 @@ def check_method_for_perf_regressions(func: Callable) -> Callable: hist = perfdb.get(func_id, []) if len(hist) < config.config['unittests_num_perf_samples']: hist.append(run_time) - logger.debug( - f'Still establishing a perf baseline for {func_name}' - ) + logger.debug(f'Still establishing a perf baseline for {func_name}') else: stdev = statistics.stdev(hist) logger.debug(f'For {func_name}, performance stdev={stdev}') slowest = hist[-1] logger.debug(f'For {func_name}, slowest perf on record is {slowest:f}s') limit = slowest + stdev * 4 - logger.debug( - f'For {func_name}, max acceptable runtime is {limit:f}s' - ) - logger.debug( - f'For {func_name}, actual observed runtime was {run_time:f}s' - ) - if ( - run_time > limit and - not config.config['unittests_ignore_perf'] - ): + logger.debug(f'For {func_name}, max acceptable runtime is {limit:f}s') + logger.debug(f'For {func_name}, actual observed runtime was {run_time:f}s') + if run_time > limit: msg = f'''{func_id} performance has regressed unacceptably. {slowest:f}s is the slowest runtime on record in {len(hist)} perf samples. It just ran in {run_time:f}s which is 4+ stdevs slower than the slowest. @@ -226,8 +221,8 @@ Here is the current, full db perf timing distribution: for x in hist: msg += f'{x:f}\n' logger.error(msg) - slf = args[0] # Peek at the wrapped function's self ref. - slf.fail(msg) # ...to fail the testcase. + slf = args[0] # Peek at the wrapped function's self ref. + slf.fail(msg) # ...to fail the testcase. else: hist.append(run_time) @@ -239,6 +234,7 @@ Here is the current, full db perf timing distribution: perfdb[func_id] = hist helper.save_performance_data(func_id, perfdb) return value + return wrapper_perf_monitor @@ -255,6 +251,7 @@ def check_all_methods_for_perf_regressions(prefix='test_'): ... """ + def decorate_the_testcase(cls): if issubclass(cls, unittest.TestCase): for name, m in inspect.getmembers(cls, inspect.isfunction): @@ -262,12 +259,14 @@ def check_all_methods_for_perf_regressions(prefix='test_'): setattr(cls, name, check_method_for_perf_regressions(m)) logger.debug(f'Wrapping {cls.__name__}:{name}.') return cls + return decorate_the_testcase def breakpoint(): """Hard code a breakpoint somewhere; drop into pdb.""" import pdb + pdb.set_trace() @@ -279,18 +278,21 @@ class RecordStdout(object): ... print("This is a test!") >>> print({record().readline()}) {'This is a test!\\n'} + >>> record().close() """ def __init__(self) -> None: self.destination = tempfile.SpooledTemporaryFile(mode='r+') - self.recorder = None + self.recorder: Optional[contextlib.redirect_stdout] = None def __enter__(self) -> Callable[[], tempfile.SpooledTemporaryFile]: self.recorder = contextlib.redirect_stdout(self.destination) + assert self.recorder self.recorder.__enter__() return lambda: self.destination - def __exit__(self, *args) -> bool: + def __exit__(self, *args) -> Optional[bool]: + assert self.recorder self.recorder.__exit__(*args) self.destination.seek(0) return None @@ -305,18 +307,21 @@ class RecordStderr(object): ... print("This is a test!", file=sys.stderr) >>> print({record().readline()}) {'This is a test!\\n'} + >>> record().close() """ def __init__(self) -> None: self.destination = tempfile.SpooledTemporaryFile(mode='r+') - self.recorder = None + self.recorder: Optional[contextlib.redirect_stdout[Any]] = None def __enter__(self) -> Callable[[], tempfile.SpooledTemporaryFile]: - self.recorder = contextlib.redirect_stderr(self.destination) + self.recorder = contextlib.redirect_stderr(self.destination) # type: ignore + assert self.recorder self.recorder.__enter__() return lambda: self.destination - def __exit__(self, *args) -> bool: + def __exit__(self, *args) -> Optional[bool]: + assert self.recorder self.recorder.__exit__(*args) self.destination.seek(0) return None @@ -330,7 +335,7 @@ class RecordMultipleStreams(object): def __init__(self, *files) -> None: self.files = [*files] self.destination = tempfile.SpooledTemporaryFile(mode='r+') - self.saved_writes = [] + self.saved_writes: List[Callable[..., Any]] = [] def __enter__(self) -> Callable[[], tempfile.SpooledTemporaryFile]: for f in self.files: @@ -338,12 +343,14 @@ class RecordMultipleStreams(object): f.write = self.destination.write return lambda: self.destination - def __exit__(self, *args) -> bool: + def __exit__(self, *args) -> Optional[bool]: for f in self.files: f.write = self.saved_writes.pop() self.destination.seek(0) + return None if __name__ == '__main__': import doctest + doctest.testmod()