X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=unittest_utils.py;h=4a9669d3a21f66e35004e1968cc85b65d711fd5c;hb=36fea7f15ed17150691b5b3ead75450e575229ef;hp=584eb3ce002d3fed3363223cbe4a3cf9059313a6;hpb=ff0c10dff77a141bfebcce592eef34d6b065bfa9;p=python_utils.git diff --git a/unittest_utils.py b/unittest_utils.py index 584eb3c..4a9669d 100644 --- a/unittest_utils.py +++ b/unittest_utils.py @@ -24,6 +24,7 @@ import warnings import bootstrap import config +import function_utils import scott_secrets import sqlalchemy as sa @@ -31,8 +32,8 @@ import sqlalchemy as sa logger = logging.getLogger(__name__) cfg = config.add_commandline_args( - f'Logging ({__file__})', - 'Args related to function decorators') + f'Logging ({__file__})', 'Args related to function decorators' +) cfg.add_argument( '--unittests_ignore_perf', action='store_true', @@ -43,34 +44,34 @@ cfg.add_argument( '--unittests_num_perf_samples', type=int, default=50, - help='The count of perf timing samples we need to see before blocking slow runs on perf grounds' + help='The count of perf timing samples we need to see before blocking slow runs on perf grounds', ) cfg.add_argument( '--unittests_drop_perf_traces', type=str, nargs=1, default=None, - help='The identifier (i.e. file!test_fixture) for which we should drop all perf data' + help='The identifier (i.e. file!test_fixture) for which we should drop all perf data', ) cfg.add_argument( '--unittests_persistance_strategy', choices=['FILE', 'DATABASE'], default='DATABASE', - help='Should we persist perf data in a file or db?' + help='Should we persist perf data in a file or db?', ) cfg.add_argument( '--unittests_perfdb_filename', type=str, metavar='FILENAME', default=f'{os.environ["HOME"]}/.python_unittest_performance_db', - help='File in which to store perf data (iff --unittests_persistance_strategy is FILE)' + help='File in which to store perf data (iff --unittests_persistance_strategy is FILE)', ) cfg.add_argument( '--unittests_perfdb_spec', type=str, metavar='DBSPEC', default='mariadb+pymysql://python_unittest:@db.house:3306/python_unittest_performance', - help='Db connection spec for perf data (iff --unittest_persistance_strategy is DATABASE)' + help='Db connection spec for perf data (iff --unittest_persistance_strategy is DATABASE)', ) # >>> This is the hacky business, FYI. <<< @@ -86,7 +87,9 @@ class PerfRegressionDataPersister(ABC): pass @abstractmethod - def save_performance_data(self, method_id: str, data: Dict[str, List[float]]): + def save_performance_data( + self, method_id: str, data: Dict[str, List[float]] + ): pass @abstractmethod @@ -103,7 +106,9 @@ class FileBasedPerfRegressionDataPersister(PerfRegressionDataPersister): with open(self.filename, 'rb') as f: return pickle.load(f) - def save_performance_data(self, method_id: str, data: Dict[str, List[float]]): + def save_performance_data( + self, method_id: str, data: Dict[str, List[float]] + ): for trace in self.traces_to_delete: if trace in data: data[trace] = [] @@ -133,7 +138,9 @@ class DatabasePerfRegressionDataPersister(PerfRegressionDataPersister): results.close() return ret - def save_performance_data(self, method_id: str, data: Dict[str, List[float]]): + def save_performance_data( + self, method_id: str, data: Dict[str, List[float]] + ): self.delete_performance_data(method_id) for (method_id, perf_data) in data.items(): sql = 'INSERT INTO runtimes_by_function (function, runtime) VALUES ' @@ -145,29 +152,6 @@ class DatabasePerfRegressionDataPersister(PerfRegressionDataPersister): self.conn.execute(sql) -def function_identifier(f: Callable) -> str: - """ - Given a callable function, return a string that identifies it. - Usually that string is just __module__:__name__ but there's a - corner case: when __module__ is __main__ (i.e. the callable is - defined in the same module as __main__). In this case, - f.__module__ returns "__main__" instead of the file that it is - defined in. Work around this using pathlib.Path (see below). - - >>> function_identifier(function_identifier) - 'unittest_utils:function_identifier' - - """ - if f.__module__ == '__main__': - from pathlib import Path - import __main__ - module = __main__.__file__ - module = Path(module).stem - return f'{module}:{f.__name__}' - else: - return f'{f.__module__}:{f.__name__}' - - def check_method_for_perf_regressions(func: Callable) -> Callable: """ This is meant to be used on a method in a class that subclasses @@ -177,6 +161,7 @@ def check_method_for_perf_regressions(func: Callable) -> Callable: message if it has become too slow. """ + @functools.wraps(func) def wrapper_perf_monitor(*args, **kwargs): if config.config['unittests_persistance_strategy'] == 'FILE': @@ -184,15 +169,18 @@ def check_method_for_perf_regressions(func: Callable) -> Callable: helper = FileBasedPerfRegressionDataPersister(filename) elif config.config['unittests_persistance_strategy'] == 'DATABASE': dbspec = config.config['unittests_perfdb_spec'] - dbspec = dbspec.replace('', scott_secrets.MARIADB_UNITTEST_PERF_PASSWORD) + dbspec = dbspec.replace( + '', scott_secrets.MARIADB_UNITTEST_PERF_PASSWORD + ) helper = DatabasePerfRegressionDataPersister(dbspec) else: raise Exception( 'Unknown/unexpected --unittests_persistance_strategy value' ) - logger.debug(f'Watching {func.__name__}\'s performance...') - func_id = function_identifier(func) + func_id = function_utils.function_identifier(func) + func_name = func.__name__ + logger.debug(f'Watching {func_name}\'s performance...') logger.debug(f'Canonical function identifier = {func_id}') try: @@ -219,49 +207,62 @@ def check_method_for_perf_regressions(func: Callable) -> Callable: hist = perfdb.get(func_id, []) if len(hist) < config.config['unittests_num_perf_samples']: hist.append(run_time) - logger.debug( - f'Still establishing a perf baseline for {func.__name__}' - ) + logger.debug(f'Still establishing a perf baseline for {func_name}') else: stdev = statistics.stdev(hist) - logger.debug(f'For {func.__name__}, performance stdev={stdev}') + logger.debug(f'For {func_name}, performance stdev={stdev}') slowest = hist[-1] - logger.debug(f'For {func.__name__}, slowest perf on record is {slowest:f}s') + logger.debug( + f'For {func_name}, slowest perf on record is {slowest:f}s' + ) limit = slowest + stdev * 4 logger.debug( - f'For {func.__name__}, max acceptable runtime is {limit:f}s' + f'For {func_name}, max acceptable runtime is {limit:f}s' ) logger.debug( - f'For {func.__name__}, actual observed runtime was {run_time:f}s' + f'For {func_name}, actual observed runtime was {run_time:f}s' ) - if ( - run_time > limit and - not config.config['unittests_ignore_perf'] - ): + if run_time > limit and not config.config['unittests_ignore_perf']: msg = f'''{func_id} performance has regressed unacceptably. -{hist[-1]:f}s is the slowest record in {len(hist)} db perf samples. -It just ran in {run_time:f}s which is >5 stdevs slower than the slowest sample. +{slowest:f}s is the slowest runtime on record in {len(hist)} perf samples. +It just ran in {run_time:f}s which is 4+ stdevs slower than the slowest. Here is the current, full db perf timing distribution: ''' for x in hist: msg += f'{x:f}\n' logger.error(msg) - slf = args[0] - slf.fail(msg) + slf = args[0] # Peek at the wrapped function's self ref. + slf.fail(msg) # ...to fail the testcase. else: hist.append(run_time) + # Don't spam the database with samples; just pick a random + # sample from what we have and store that back. n = min(config.config['unittests_num_perf_samples'], len(hist)) hist = random.sample(hist, n) hist.sort() perfdb[func_id] = hist helper.save_performance_data(func_id, perfdb) return value + return wrapper_perf_monitor def check_all_methods_for_perf_regressions(prefix='test_'): + """Decorate unittests with this to pay attention to the perf of the + testcode and flag perf regressions. e.g. + + import unittest_utils as uu + + @uu.check_all_methods_for_perf_regressions() + class TestMyClass(unittest.TestCase): + + def test_some_part_of_my_class(self): + ... + + """ + def decorate_the_testcase(cls): if issubclass(cls, unittest.TestCase): for name, m in inspect.getmembers(cls, inspect.isfunction): @@ -269,12 +270,14 @@ def check_all_methods_for_perf_regressions(prefix='test_'): setattr(cls, name, check_method_for_perf_regressions(m)) logger.debug(f'Wrapping {cls.__name__}:{name}.') return cls + return decorate_the_testcase def breakpoint(): """Hard code a breakpoint somewhere; drop into pdb.""" import pdb + pdb.set_trace() @@ -353,4 +356,5 @@ class RecordMultipleStreams(object): if __name__ == '__main__': import doctest + doctest.testmod()