8 from inspect import stack
12 from argparse_utils import ActionNoYes
14 # This module is commonly used by others in here and should avoid
15 # taking any unnecessary dependencies back on them.
18 logger = logging.getLogger(__name__)
20 cfg = config.add_commandline_args(
21 f'Bootstrap ({__file__})',
22 'Args related to python program bootstrapper and Swiss army knife',
25 '--debug_unhandled_exceptions',
28 help='Break into pdb on top level unhandled exceptions.',
34 help='Should we display (and log.debug) the global random seed?',
42 help='Override the global random seed with a particular number.',
48 help='Should we dump the Python import tree before main?',
51 '--audit_import_events',
54 help='Should we audit all import events?',
60 help='Should we run cProfile on this code?',
66 help='Should we record/report on memory utilization?',
69 ORIGINAL_EXCEPTION_HOOK = sys.excepthook
72 def handle_uncaught_exception(exc_type, exc_value, exc_tb):
74 Top-level exception handler for exceptions that make it past any exception
75 handlers in the python code being run. Logs the error and stacktrace then
76 maybe attaches a debugger.
79 msg = f'Unhandled top level exception {exc_type}'
81 print(msg, file=sys.stderr)
82 if issubclass(exc_type, KeyboardInterrupt):
83 sys.__excepthook__(exc_type, exc_value, exc_tb)
86 if not sys.stderr.isatty() or not sys.stdin.isatty():
87 # stdin or stderr is redirected, just do the normal thing
88 ORIGINAL_EXCEPTION_HOOK(exc_type, exc_value, exc_tb)
90 # a terminal is attached and stderr is not redirected, maybe debug.
93 traceback.print_exception(exc_type, exc_value, exc_tb)
94 if config.config['debug_unhandled_exceptions']:
97 logger.info("Invoking the debugger...")
100 ORIGINAL_EXCEPTION_HOOK(exc_type, exc_value, exc_tb)
103 class ImportInterceptor(importlib.abc.MetaPathFinder):
107 self.module_by_filename_cache = {}
108 self.repopulate_modules_by_filename()
109 self.tree = collect.trie.Trie()
110 self.tree_node_by_module = {}
112 def repopulate_modules_by_filename(self):
113 self.module_by_filename_cache.clear()
114 for mod in sys.modules:
115 if hasattr(sys.modules[mod], '__file__'):
116 fname = getattr(sys.modules[mod], '__file__')
119 self.module_by_filename_cache[fname] = mod
122 def should_ignore_filename(filename: str) -> bool:
123 return 'importlib' in filename or 'six.py' in filename
125 def find_module(self, fullname, path):
126 raise Exception("This method has been deprecated since Python 3.4, please upgrade.")
128 def find_spec(self, loaded_module, path=None, _=None):
130 for x in range(3, len(s)):
131 filename = s[x].filename
132 if ImportInterceptor.should_ignore_filename(filename):
135 loading_function = s[x].function
136 if filename in self.module_by_filename_cache:
137 loading_module = self.module_by_filename_cache[filename]
139 self.repopulate_modules_by_filename()
140 loading_module = self.module_by_filename_cache.get(filename, 'unknown')
142 path = self.tree_node_by_module.get(loading_module, [])
143 path.extend([loaded_module])
144 self.tree.insert(path)
145 self.tree_node_by_module[loading_module] = path
147 msg = f'*** Import {loaded_module} from {filename}:{s[x].lineno} in {loading_module}::{loading_function}'
151 msg = f'*** Import {loaded_module} from ?????'
155 def invalidate_caches(self):
158 def find_importer(self, module: str):
159 if module in self.tree_node_by_module:
160 node = self.tree_node_by_module[module]
165 # Audit import events? Note: this runs early in the lifetime of the
166 # process (assuming that import bootstrap happens early); config has
167 # (probably) not yet been loaded or parsed the commandline. Also,
168 # some things have probably already been imported while we weren't
169 # watching so this information may be incomplete.
171 # Also note: move bootstrap up in the global import list to catch
172 # more import events and have a more complete record.
173 IMPORT_INTERCEPTOR = None
175 if arg == '--audit_import_events':
176 IMPORT_INTERCEPTOR = ImportInterceptor()
177 sys.meta_path.insert(0, IMPORT_INTERCEPTOR)
180 def dump_all_objects() -> None:
182 all_modules = sys.modules
183 for obj in object.__subclasses__():
184 if not hasattr(obj, '__name__'):
187 if not hasattr(obj, '__module__'):
189 class_mod_name = obj.__module__
190 if class_mod_name in all_modules:
191 mod = all_modules[class_mod_name]
192 if not hasattr(mod, '__name__'):
193 mod_name = class_mod_name
195 mod_name = mod.__name__
196 if hasattr(mod, '__file__'):
197 mod_file = mod.__file__
200 if IMPORT_INTERCEPTOR is not None:
201 import_path = IMPORT_INTERCEPTOR.find_importer(mod_name)
203 import_path = 'unknown'
204 msg = f'{class_mod_name}::{klass} ({mod_file})'
205 if import_path != 'unknown' and len(import_path) > 0:
206 msg += f' imported by {import_path}'
207 messages[f'{class_mod_name}::{klass}'] = msg
208 for x in sorted(messages.keys()):
209 logger.debug(messages[x])
213 def initialize(entry_point):
215 Remember to initialize config, initialize logging, set/log a random
216 seed, etc... before running main.
220 @functools.wraps(entry_point)
221 def initialize_wrapper(*args, **kwargs):
222 # Hook top level unhandled exceptions, maybe invoke debugger.
223 if sys.excepthook == sys.__excepthook__:
224 sys.excepthook = handle_uncaught_exception
226 # Try to figure out the name of the program entry point. Then
227 # parse configuration (based on cmdline flags, environment vars
229 if '__globals__' in entry_point.__dict__ and '__file__' in entry_point.__globals__:
230 config.parse(entry_point.__globals__['__file__'])
234 if config.config['trace_memory']:
239 # Initialize logging... and log some remembered messages from
241 logging_utils.initialize_logging(logging.getLogger())
242 config.late_logging()
244 # Maybe log some info about the python interpreter itself.
246 'Platform: %s, maxint=0x%x, byteorder=%s',
247 sys.platform, sys.maxsize, sys.byteorder
249 logger.debug('Python interpreter version: %s', sys.version)
250 logger.debug('Python implementation: %s', sys.implementation)
251 logger.debug('Python C API version: %s', sys.api_version)
252 logger.debug('Python path: %s', sys.path)
254 # Log something about the site_config, many things use it.
257 logger.debug('Global site_config: %s', site_config.get_config())
259 # Allow programs that don't bother to override the random seed
260 # to be replayed via the commandline.
263 random_seed = config.config['set_random_seed']
264 if random_seed is not None:
265 random_seed = random_seed[0]
267 random_seed = int.from_bytes(os.urandom(4), 'little')
269 if config.config['show_random_seed']:
270 msg = f'Global random seed is: {random_seed}'
273 random.seed(random_seed)
275 # Do it, invoke the user's code. Pay attention to how long it takes.
276 logger.debug('Starting %s (program entry point)', entry_point.__name__)
280 if config.config['run_profiler']:
282 from pstats import SortKey
284 with stopwatch.Timer() as t:
286 "ret = entry_point(*args, **kwargs)",
293 with stopwatch.Timer() as t:
294 ret = entry_point(*args, **kwargs)
296 logger.debug('%s (program entry point) returned %s.', entry_point.__name__, ret)
298 if config.config['trace_memory']:
299 snapshot = tracemalloc.take_snapshot()
300 top_stats = snapshot.statistics('lineno')
302 print("--trace_memory's top 10 memory using files:")
303 for stat in top_stats[:10]:
306 if config.config['dump_all_objects']:
309 if config.config['audit_import_events']:
310 if IMPORT_INTERCEPTOR is not None:
311 print(IMPORT_INTERCEPTOR.tree)
314 (utime, stime, cutime, cstime, elapsed_time) = os.times()
319 'child user: %.4fs\n'
320 'child system: %.4fs\n'
321 'machine uptime: %.4fs\n'
323 utime, stime, cutime, cstime, elapsed_time, walltime
326 # If it doesn't return cleanly, call attention to the return value.
327 if ret is not None and ret != 0:
328 logger.error('Exit %s', ret)
330 logger.debug('Exit %s', ret)
333 return initialize_wrapper