Optionally surface exceptions that happen under executors by reading

[python_utils.git] / bootstrap.py

diff --git a/bootstrap.py b/bootstrap.py
index c3b70db106260bbc80b592f45ea1fc70e6df6254..e50cb386543aaef66fdd36cb9c149fbf009ef5ae 100644 (file)
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -51,7 +51,18 @@ args.add_argument(
     default=False,
     help='Should we audit all import events?',
 )
-
+args.add_argument(
+    '--run_profiler',
+    action=ActionNoYes,
+    default=False,
+    help='Should we run cProfile on this code?',
+)
+args.add_argument(
+    '--trace_memory',
+    action=ActionNoYes,
+    default=False,
+    help='Should we record/report on memory utilization?',
+)
 
 original_hook = sys.excepthook
 
@@ -121,9 +132,7 @@ class ImportInterceptor(object):
                 loading_module = self.module_by_filename_cache[filename]
             else:
                 self.repopulate_modules_by_filename()
-                loading_module = self.module_by_filename_cache.get(
-                    filename, 'unknown'
-                )
+                loading_module = self.module_by_filename_cache.get(filename, 'unknown')
 
             path = self.tree_node_by_module.get(loading_module, [])
             path.extend([loaded_module])
@@ -145,18 +154,6 @@ class ImportInterceptor(object):
         return []
 
 
-# # TODO: test this with python 3.8+
-# def audit_import_events(event, args):
-#     if event == 'import':
-#         module = args[0]
-#         filename = args[1]
-#         sys_path = args[2]
-#         sys_meta_path = args[3]
-#         sys_path_hooks = args[4]
-#         logger.debug(msg)
-#         print(msg)
-
-
 # Audit import events?  Note: this runs early in the lifetime of the
 # process (assuming that import bootstrap happens early); config has
 # (probably) not yet been loaded or parsed the commandline.  Also,
@@ -170,12 +167,6 @@ for arg in sys.argv:
     if arg == '--audit_import_events':
         import_interceptor = ImportInterceptor()
         sys.meta_path = [import_interceptor] + sys.meta_path
-        # if not hasattr(sys, 'frozen'):
-        #     if (
-        #             sys.version_info[0] == 3
-        #             and sys.version_info[1] >= 8
-        #     ):
-        #         sys.addaudithook(audit_import_events)
 
 
 def dump_all_objects() -> None:
@@ -236,11 +227,25 @@ def initialize(entry_point):
         else:
             config.parse(None)
 
+        if config.config['trace_memory']:
+            import tracemalloc
+
+            tracemalloc.start()
+
         # Initialize logging... and log some remembered messages from
         # config module.
         logging_utils.initialize_logging(logging.getLogger())
         config.late_logging()
 
+        # Maybe log some info about the python interpreter itself.
+        logger.debug(
+            f'Platform: {sys.platform}, maxint=0x{sys.maxsize:x}, byteorder={sys.byteorder}'
+        )
+        logger.debug(f'Python interpreter version: {sys.version}')
+        logger.debug(f'Python implementation: {sys.implementation}')
+        logger.debug(f'Python C API version: {sys.api_version}')
+        logger.debug(f'Python path: {sys.path}')
+
         # Allow programs that don't bother to override the random seed
         # to be replayed via the commandline.
         import random
@@ -262,11 +267,31 @@ def initialize(entry_point):
         ret = None
         import stopwatch
 
-        with stopwatch.Timer() as t:
-            ret = entry_point(*args, **kwargs)
-        logger.debug(
-            f'{entry_point.__name__} (program entry point) returned {ret}.'
-        )
+        if config.config['run_profiler']:
+            import cProfile
+            from pstats import SortKey
+
+            with stopwatch.Timer() as t:
+                cProfile.runctx(
+                    "ret = entry_point(*args, **kwargs)",
+                    globals(),
+                    locals(),
+                    None,
+                    SortKey.CUMULATIVE,
+                )
+        else:
+            with stopwatch.Timer() as t:
+                ret = entry_point(*args, **kwargs)
+
+        logger.debug(f'{entry_point.__name__} (program entry point) returned {ret}.')
+
+        if config.config['trace_memory']:
+            snapshot = tracemalloc.take_snapshot()
+            top_stats = snapshot.statistics('lineno')
+            print()
+            print("--trace_memory's top 10 memory using files:")
+            for stat in top_stats[:10]:
+                print(stat)
 
         if config.config['dump_all_objects']:
             dump_all_objects()