From 59cae873b00e3459d9a7f794398894c95ab682ed Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Fri, 3 Jun 2022 10:14:01 -0700 Subject: [PATCH] Make run_tests.py also run a subset of tests w/o coverage (in addition to with coverage) when --coverage is passed so that they can pay attention to perf regressions. --- tests/run_tests.py | 192 ++++++++++++++++++++++++++++++++------------- 1 file changed, 137 insertions(+), 55 deletions(-) diff --git a/tests/run_tests.py b/tests/run_tests.py index fa75539..c96f882 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -43,6 +43,12 @@ args.add_argument( HOME = os.environ['HOME'] +# These tests will be run twice in --coverage mode: once to get code +# coverage and then again with not coverage enabeled. This is because +# they pay attention to code performance which is adversely affected +# by coverage. +PERF_SENSATIVE_TESTS = set(['/home/scott/lib/python_modules/tests/string_utils_test.py']) + @dataclass class TestingParameters: @@ -53,6 +59,18 @@ class TestingParameters: """An event that, when set, indicates to stop ASAP.""" +@dataclass +class TestToRun: + name: str + """The name of the test""" + + kind: str + """The kind of the test""" + + cmdline: str + """The command line to execute""" + + @dataclass class TestResults: name: str @@ -143,12 +161,12 @@ class TemplatedTestRunner(TestRunner, ABC): """A TestRunner that has a recipe for executing the tests.""" @abstractmethod - def identify_tests(self) -> List[str]: - """Return a list of tests that should be executed.""" + def identify_tests(self) -> List[TestToRun]: + """Return a list of tuples (test, cmdline) that should be executed.""" pass @abstractmethod - def run_test(self, test: Any) -> TestResults: + def run_test(self, test: TestToRun) -> TestResults: """Run a single test and return its TestResults.""" pass @@ -163,11 +181,10 @@ class TemplatedTestRunner(TestRunner, ABC): logger.error('Thread %s saw abnormal results; exiting.', self.get_name()) raise Exception("Kill myself!") - def persist_output(self, test_name: str, message: str, output: str) -> None: + def persist_output(self, test: TestToRun, message: str, output: str) -> None: """Called to save the output of a test run.""" - basename = file_utils.without_path(test_name) - dest = f'{basename}-output.txt' + dest = f'{test.name}-output.txt' with open(f'./test_output/{dest}', 'w') as wf: print(message, file=wf) print('-' * len(message), file=wf) @@ -175,46 +192,46 @@ class TemplatedTestRunner(TestRunner, ABC): def execute_commandline( self, - test_name: str, - cmdline: str, + test: TestToRun, *, timeout: float = 120.0, ) -> TestResults: """Execute a particular commandline to run a test.""" try: - logger.debug('%s: Running %s (%s)', self.get_name(), test_name, cmdline) output = exec_utils.cmd( - cmdline, + test.cmdline, timeout_seconds=timeout, ) - self.persist_output(test_name, f'{test_name} ({cmdline}) succeeded.', output) - logger.debug('%s (%s) succeeded', test_name, cmdline) - return TestResults(test_name, [test_name], [test_name], [], []) + self.persist_output(test, f'{test.name} ({test.cmdline}) succeeded.', output) + logger.debug('%s: %s (%s) succeeded', self.get_name(), test.name, test.cmdline) + return TestResults(test.name, [test.name], [test.name], [], []) except subprocess.TimeoutExpired as e: - msg = f'{self.get_name()}: {test_name} ({cmdline}) timed out after {e.timeout:.1f} seconds.' + msg = f'{self.get_name()}: {test.name} ({test.cmdline}) timed out after {e.timeout:.1f} seconds.' logger.error(msg) logger.debug( - '%s: %s output when it timed out: %s', self.get_name(), test_name, e.output + '%s: %s output when it timed out: %s', self.get_name(), test.name, e.output ) - self.persist_output(test_name, msg, e.output.decode('utf-8')) + self.persist_output(test, msg, e.output.decode('utf-8')) return TestResults( - test_name, - [test_name], + test.name, + [test.name], [], [], - [test_name], + [test.name], ) except subprocess.CalledProcessError as e: - msg = f'{self.get_name()}: {test_name} ({cmdline}) failed; exit code {e.returncode}' + msg = ( + f'{self.get_name()}: {test.name} ({test.cmdline}) failed; exit code {e.returncode}' + ) logger.error(msg) - logger.debug('%s: %s output when it failed: %s', self.get_name(), test_name, e.output) - self.persist_output(test_name, msg, e.output.decode('utf-8')) + logger.debug('%s: %s output when it failed: %s', self.get_name(), test.name, e.output) + self.persist_output(test, msg, e.output.decode('utf-8')) return TestResults( - test_name, - [test_name], + test.name, + [test.name], [], - [test_name], + [test.name], [], ) @@ -222,11 +239,18 @@ class TemplatedTestRunner(TestRunner, ABC): def begin(self, params: TestingParameters) -> TestResults: logger.debug('Thread %s started.', self.get_name()) interesting_tests = self.identify_tests() + logger.debug('%s: Identified %d tests to be run.', self.get_name(), len(interesting_tests)) + # Note: because of @parallelize on run_tests it actually + # returns a SmartFuture with a TestResult inside of it. + # That's the reason for this Any business. running: List[Any] = [] - for test in interesting_tests: - running.append(self.run_test(test)) - self.tests_started = len(running) + for test_to_run in interesting_tests: + running.append(self.run_test(test_to_run)) + logger.debug( + '%s: Test %s started in the background.', self.get_name(), test_to_run.name + ) + self.tests_started += 1 for future in smart_future.wait_any(running): self.check_for_abort() @@ -246,16 +270,39 @@ class UnittestTestRunner(TemplatedTestRunner): return "Unittests" @overrides - def identify_tests(self) -> List[str]: - return list(file_utils.expand_globs('*_test.py')) + def identify_tests(self) -> List[TestToRun]: + ret = [] + for test in file_utils.expand_globs('*_test.py'): + basename = file_utils.without_path(test) + if config.config['coverage']: + ret.append( + TestToRun( + name=basename, + kind='unittest capturing coverage', + cmdline=f'coverage run --source {HOME}/lib {test} --unittests_ignore_perf 2>&1', + ) + ) + if test in PERF_SENSATIVE_TESTS: + ret.append( + TestToRun( + name=basename, + kind='unittest w/o coverage to record perf', + cmdline=f'{test} 2>&1', + ) + ) + else: + ret.append( + TestToRun( + name=basename, + kind='unittest', + cmdline=f'{test} 2>&1', + ) + ) + return ret @par.parallelize - def run_test(self, test: Any) -> TestResults: - if config.config['coverage']: - cmdline = f'coverage run --source {HOME}/lib {test} --unittests_ignore_perf' - else: - cmdline = test - return self.execute_commandline(test, cmdline) + def run_test(self, test: TestToRun) -> TestResults: + return self.execute_commandline(test) class DoctestTestRunner(TemplatedTestRunner): @@ -266,22 +313,38 @@ class DoctestTestRunner(TemplatedTestRunner): return "Doctests" @overrides - def identify_tests(self) -> List[str]: + def identify_tests(self) -> List[TestToRun]: ret = [] out = exec_utils.cmd('grep -lR "^ *import doctest" /home/scott/lib/python_modules/*') - for line in out.split('\n'): - if re.match(r'.*\.py$', line): - if 'run_tests.py' not in line: - ret.append(line) + for test in out.split('\n'): + if re.match(r'.*\.py$', test): + if 'run_tests.py' not in test: + basename = file_utils.without_path(test) + if config.config['coverage']: + ret.append( + TestToRun( + name=basename, + kind='doctest capturing coverage', + cmdline=f'coverage run --source {HOME}/lib {test} 2>&1', + ) + ) + if test in PERF_SENSATIVE_TESTS: + ret.append( + TestToRun( + name=basename, + kind='doctest w/o coverage to record perf', + cmdline=f'python3 {test} 2>&1', + ) + ) + else: + ret.append( + TestToRun(name=basename, kind='doctest', cmdline=f'python3 {test} 2>&1') + ) return ret @par.parallelize - def run_test(self, test: Any) -> TestResults: - if config.config['coverage']: - cmdline = f'coverage run --source {HOME}/lib {test} 2>&1' - else: - cmdline = f'python3 {test}' - return self.execute_commandline(test, cmdline) + def run_test(self, test: TestToRun) -> TestResults: + return self.execute_commandline(test) class IntegrationTestRunner(TemplatedTestRunner): @@ -292,16 +355,35 @@ class IntegrationTestRunner(TemplatedTestRunner): return "Integration Tests" @overrides - def identify_tests(self) -> List[str]: - return list(file_utils.expand_globs('*_itest.py')) + def identify_tests(self) -> List[TestToRun]: + ret = [] + for test in file_utils.expand_globs('*_itest.py'): + basename = file_utils.without_path(test) + if config.config['coverage']: + ret.append( + TestToRun( + name=basename, + kind='integration test capturing coverage', + cmdline=f'coverage run --source {HOME}/lib {test} 2>&1', + ) + ) + if test in PERF_SENSATIVE_TESTS: + ret.append( + TestToRun( + name=basename, + kind='integration test w/o coverage to capture perf', + cmdline=f'{test} 2>&1', + ) + ) + else: + ret.append( + TestToRun(name=basename, kind='integration test', cmdline=f'{test} 2>&1') + ) + return ret @par.parallelize - def run_test(self, test: Any) -> TestResults: - if config.config['coverage']: - cmdline = f'coverage run --source {HOME}/lib {test}' - else: - cmdline = test - return self.execute_commandline(test, cmdline) + def run_test(self, test: TestToRun) -> TestResults: + return self.execute_commandline(test) def test_results_report(results: Dict[str, TestResults]) -> int: -- 2.45.2