Used isort to sort imports. Also added to the git pre-commit hook.

[python_utils.git] / bootstrap.py

diff --git a/bootstrap.py b/bootstrap.py
index 44a16fbd3b3f99a20e47913f0428314b1cf5bcc8..035a38eca691ca7255a4ee4bda0eee6da0813119 100644 (file)
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -1,17 +1,20 @@
 #!/usr/bin/env python3
 
 import functools
+import importlib
 import logging
 import os
-from inspect import stack
 import sys
+from inspect import stack
+from typing import List
+
+import config
+import logging_utils
+from argparse_utils import ActionNoYes
 
 # This module is commonly used by others in here and should avoid
 # taking any unnecessary dependencies back on them.
 
-from argparse_utils import ActionNoYes
-import config
-import logging_utils
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +54,18 @@ args.add_argument(
     default=False,
     help='Should we audit all import events?',
 )
-
+args.add_argument(
+    '--run_profiler',
+    action=ActionNoYes,
+    default=False,
+    help='Should we run cProfile on this code?',
+)
+args.add_argument(
+    '--trace_memory',
+    action=ActionNoYes,
+    default=False,
+    help='Should we record/report on memory utilization?',
+)
 
 original_hook = sys.excepthook
 
@@ -88,7 +102,7 @@ def handle_uncaught_exception(exc_type, exc_value, exc_tb):
                 original_hook(exc_type, exc_value, exc_tb)
 
 
-class ImportInterceptor(object):
+class ImportInterceptor(importlib.abc.MetaPathFinder):
     def __init__(self):
         import collect.trie
 
@@ -109,6 +123,11 @@ class ImportInterceptor(object):
     def should_ignore_filename(self, filename: str) -> bool:
         return 'importlib' in filename or 'six.py' in filename
 
+    def find_module(self, fullname, path):
+        raise Exception(
+            "This method has been deprecated since Python 3.4, please upgrade."
+        )
+
     def find_spec(self, loaded_module, path=None, target=None):
         s = stack()
         for x in range(3, len(s)):
@@ -136,6 +155,9 @@ class ImportInterceptor(object):
         logger.debug(msg)
         print(msg)
 
+    def invalidate_caches(self):
+        pass
+
     def find_importer(self, module: str):
         if module in self.tree_node_by_module:
             node = self.tree_node_by_module[module]
@@ -143,18 +165,6 @@ class ImportInterceptor(object):
         return []
 
 
-# # TODO: test this with python 3.8+
-# def audit_import_events(event, args):
-#     if event == 'import':
-#         module = args[0]
-#         filename = args[1]
-#         sys_path = args[2]
-#         sys_meta_path = args[3]
-#         sys_path_hooks = args[4]
-#         logger.debug(msg)
-#         print(msg)
-
-
 # Audit import events?  Note: this runs early in the lifetime of the
 # process (assuming that import bootstrap happens early); config has
 # (probably) not yet been loaded or parsed the commandline.  Also,
@@ -167,13 +177,7 @@ import_interceptor = None
 for arg in sys.argv:
     if arg == '--audit_import_events':
         import_interceptor = ImportInterceptor()
-        sys.meta_path = [import_interceptor] + sys.meta_path
-        # if not hasattr(sys, 'frozen'):
-        #     if (
-        #             sys.version_info[0] == 3
-        #             and sys.version_info[1] >= 8
-        #     ):
-        #         sys.addaudithook(audit_import_events)
+        sys.meta_path.insert(0, import_interceptor)
 
 
 def dump_all_objects() -> None:
@@ -234,11 +238,25 @@ def initialize(entry_point):
         else:
             config.parse(None)
 
+        if config.config['trace_memory']:
+            import tracemalloc
+
+            tracemalloc.start()
+
         # Initialize logging... and log some remembered messages from
         # config module.
         logging_utils.initialize_logging(logging.getLogger())
         config.late_logging()
 
+        # Maybe log some info about the python interpreter itself.
+        logger.debug(
+            f'Platform: {sys.platform}, maxint=0x{sys.maxsize:x}, byteorder={sys.byteorder}'
+        )
+        logger.debug(f'Python interpreter version: {sys.version}')
+        logger.debug(f'Python implementation: {sys.implementation}')
+        logger.debug(f'Python C API version: {sys.api_version}')
+        logger.debug(f'Python path: {sys.path}')
+
         # Allow programs that don't bother to override the random seed
         # to be replayed via the commandline.
         import random
@@ -260,10 +278,32 @@ def initialize(entry_point):
         ret = None
         import stopwatch
 
-        with stopwatch.Timer() as t:
-            ret = entry_point(*args, **kwargs)
+        if config.config['run_profiler']:
+            import cProfile
+            from pstats import SortKey
+
+            with stopwatch.Timer() as t:
+                cProfile.runctx(
+                    "ret = entry_point(*args, **kwargs)",
+                    globals(),
+                    locals(),
+                    None,
+                    SortKey.CUMULATIVE,
+                )
+        else:
+            with stopwatch.Timer() as t:
+                ret = entry_point(*args, **kwargs)
+
         logger.debug(f'{entry_point.__name__} (program entry point) returned {ret}.')
 
+        if config.config['trace_memory']:
+            snapshot = tracemalloc.take_snapshot()
+            top_stats = snapshot.statistics('lineno')
+            print()
+            print("--trace_memory's top 10 memory using files:")
+            for stat in top_stats[:10]:
+                print(stat)
+
         if config.config['dump_all_objects']:
             dump_all_objects()