X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=bootstrap.py;h=e50cb386543aaef66fdd36cb9c149fbf009ef5ae;hb=865825894beeedd47d26dd092d40bfee582f5475;hp=03bb505c6d3a1aed2a6177aba4fc3565d538fd17;hpb=a5c4feeac240a446a38147130b51ad96a046f6e1;p=python_utils.git diff --git a/bootstrap.py b/bootstrap.py index 03bb505..e50cb38 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import functools -import importlib import logging import os from inspect import stack @@ -18,18 +17,19 @@ logger = logging.getLogger(__name__) args = config.add_commandline_args( f'Bootstrap ({__file__})', - 'Args related to python program bootstrapper and Swiss army knife') + 'Args related to python program bootstrapper and Swiss army knife', +) args.add_argument( '--debug_unhandled_exceptions', action=ActionNoYes, default=False, - help='Break into pdb on top level unhandled exceptions.' + help='Break into pdb on top level unhandled exceptions.', ) args.add_argument( '--show_random_seed', action=ActionNoYes, default=False, - help='Should we display (and log.debug) the global random seed?' + help='Should we display (and log.debug) the global random seed?', ) args.add_argument( '--set_random_seed', @@ -37,13 +37,13 @@ args.add_argument( nargs=1, default=None, metavar='SEED_INT', - help='Override the global random seed with a particular number.' + help='Override the global random seed with a particular number.', ) args.add_argument( '--dump_all_objects', action=ActionNoYes, default=False, - help='Should we dump the Python import tree before main?' + help='Should we dump the Python import tree before main?', ) args.add_argument( '--audit_import_events', @@ -51,15 +51,28 @@ args.add_argument( default=False, help='Should we audit all import events?', ) - +args.add_argument( + '--run_profiler', + action=ActionNoYes, + default=False, + help='Should we run cProfile on this code?', +) +args.add_argument( + '--trace_memory', + action=ActionNoYes, + default=False, + help='Should we record/report on memory utilization?', +) original_hook = sys.excepthook + def handle_uncaught_exception(exc_type, exc_value, exc_tb): """ Top-level exception handler for exceptions that make it past any exception handlers in the python code being run. Logs the error and stacktrace then maybe attaches a debugger. + """ global original_hook msg = f'Unhandled top level exception {exc_type}' @@ -69,18 +82,17 @@ def handle_uncaught_exception(exc_type, exc_value, exc_tb): sys.__excepthook__(exc_type, exc_value, exc_tb) return else: - if ( - not sys.stderr.isatty() or - not sys.stdin.isatty() - ): + if not sys.stderr.isatty() or not sys.stdin.isatty(): # stdin or stderr is redirected, just do the normal thing original_hook(exc_type, exc_value, exc_tb) else: # a terminal is attached and stderr is not redirected, maybe debug. import traceback + traceback.print_exception(exc_type, exc_value, exc_tb) if config.config['debug_unhandled_exceptions']: import pdb + logger.info("Invoking the debugger...") pdb.pm() else: @@ -90,6 +102,7 @@ def handle_uncaught_exception(exc_type, exc_value, exc_tb): class ImportInterceptor(object): def __init__(self): import collect.trie + self.module_by_filename_cache = {} self.repopulate_modules_by_filename() self.tree = collect.trie.Trie() @@ -141,12 +154,6 @@ class ImportInterceptor(object): return [] -# TODO: test this with python 3.8+ -def audit_import_events(event, args): - print(event) - print(args) - - # Audit import events? Note: this runs early in the lifetime of the # process (assuming that import bootstrap happens early); config has # (probably) not yet been loaded or parsed the commandline. Also, @@ -158,15 +165,8 @@ def audit_import_events(event, args): import_interceptor = None for arg in sys.argv: if arg == '--audit_import_events': - if not hasattr(sys, 'frozen'): - if ( - sys.version_info[0] == 3 - and sys.version_info[1] < 8 - ): - import_interceptor = ImportInterceptor() - sys.meta_path = [import_interceptor] + sys.meta_path - else: - sys.addaudithook(audit_import_events) + import_interceptor = ImportInterceptor() + sys.meta_path = [import_interceptor] + sys.meta_path def dump_all_objects() -> None: @@ -209,6 +209,7 @@ def initialize(entry_point): seed, etc... before running main. """ + @functools.wraps(entry_point) def initialize_wrapper(*args, **kwargs): # Hook top level unhandled exceptions, maybe invoke debugger. @@ -219,21 +220,36 @@ def initialize(entry_point): # parse configuration (based on cmdline flags, environment vars # etc...) if ( - '__globals__' in entry_point.__dict__ and - '__file__' in entry_point.__globals__ + '__globals__' in entry_point.__dict__ + and '__file__' in entry_point.__globals__ ): config.parse(entry_point.__globals__['__file__']) else: config.parse(None) + if config.config['trace_memory']: + import tracemalloc + + tracemalloc.start() + # Initialize logging... and log some remembered messages from # config module. logging_utils.initialize_logging(logging.getLogger()) config.late_logging() + # Maybe log some info about the python interpreter itself. + logger.debug( + f'Platform: {sys.platform}, maxint=0x{sys.maxsize:x}, byteorder={sys.byteorder}' + ) + logger.debug(f'Python interpreter version: {sys.version}') + logger.debug(f'Python implementation: {sys.implementation}') + logger.debug(f'Python C API version: {sys.api_version}') + logger.debug(f'Python path: {sys.path}') + # Allow programs that don't bother to override the random seed # to be replayed via the commandline. import random + random_seed = config.config['set_random_seed'] if random_seed is not None: random_seed = random_seed[0] @@ -250,11 +266,32 @@ def initialize(entry_point): logger.debug(f'Starting {entry_point.__name__} (program entry point)') ret = None import stopwatch - with stopwatch.Timer() as t: - ret = entry_point(*args, **kwargs) - logger.debug( - f'{entry_point.__name__} (program entry point) returned {ret}.' - ) + + if config.config['run_profiler']: + import cProfile + from pstats import SortKey + + with stopwatch.Timer() as t: + cProfile.runctx( + "ret = entry_point(*args, **kwargs)", + globals(), + locals(), + None, + SortKey.CUMULATIVE, + ) + else: + with stopwatch.Timer() as t: + ret = entry_point(*args, **kwargs) + + logger.debug(f'{entry_point.__name__} (program entry point) returned {ret}.') + + if config.config['trace_memory']: + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('lineno') + print() + print("--trace_memory's top 10 memory using files:") + for stat in top_stats[:10]: + print(stat) if config.config['dump_all_objects']: dump_all_objects() @@ -266,13 +303,15 @@ def initialize(entry_point): walltime = t() (utime, stime, cutime, cstime, elapsed_time) = os.times() - logger.debug('\n' - f'user: {utime}s\n' - f'system: {stime}s\n' - f'child user: {cutime}s\n' - f'child system: {cstime}s\n' - f'machine uptime: {elapsed_time}s\n' - f'walltime: {walltime}s') + logger.debug( + '\n' + f'user: {utime}s\n' + f'system: {stime}s\n' + f'child user: {cutime}s\n' + f'child system: {cstime}s\n' + f'machine uptime: {elapsed_time}s\n' + f'walltime: {walltime}s' + ) # If it doesn't return cleanly, call attention to the return value. if ret is not None and ret != 0: @@ -280,4 +319,5 @@ def initialize(entry_point): else: logger.debug(f'Exit {ret}') sys.exit(ret) + return initialize_wrapper