Optional hooks for system wide non-zero exit log and unhandled top

level exception log.

diff --git a/src/pyutils/bootstrap.py b/src/pyutils/bootstrap.py
index 6d4e63b75a92df30d37e69659388dd6f6eb30881..f22bc1755cb30febe5aa6e3b7a99a39ff6500f69 100644 (file)
--- a/src/pyutils/bootstrap.py
+++ b/src/pyutils/bootstrap.py
@@ -118,6 +118,10 @@ def handle_uncaught_exception(exc_type, exc_value, exc_tb):
     msg = f"Unhandled top level exception {exc_type}"
     logger.exception(msg)
     print(msg, file=sys.stderr)
+    try:
+        logging_utils.unhandled_top_level_exception(exc_type)
+    except Exception:
+        pass
     if issubclass(exc_type, KeyboardInterrupt):
         sys.__excepthook__(exc_type, exc_value, exc_tb)
         return
@@ -431,6 +435,7 @@ def initialize(entry_point):
         base_filename = os.path.basename(entry_filename)
         if ret is not None and ret != 0:
             logger.error("%s: Exit %s", base_filename, ret)
+            logging_utils.non_zero_return_value(ret)
         else:
             logger.debug("%s: Exit %s", base_filename, ret)
         sys.exit(ret)

diff --git a/src/pyutils/logging_utils.py b/src/pyutils/logging_utils.py
index c99cb4e644fce847bdb0593e8df4368143e5df68..cfa674cbf5d47e5ac52c8fe99eff2079280fddf6 100644 (file)
--- a/src/pyutils/logging_utils.py
+++ b/src/pyutils/logging_utils.py
@@ -204,6 +204,20 @@ cfg.add_argument(
         + "cause you to miss logging messages."
     ),
 )
+cfg.add_argument(
+    "--logging_non_zero_exits_record_path",
+    type=str,
+    default="/var/log/abnormal_python_exits.log",
+    metavar="FILENAME",
+    help="The filename in which to record non-zero exits.",
+)
+cfg.add_argument(
+    "--logging_unhandled_top_level_exceptions_record_path",
+    type=str,
+    default="/var/log/abnormal_python_exits.log",
+    metavar="FILENAME",
+    help="The filename in which to record unhandled top level exceptions.",
+)
 
 BUILT_IN_PRINT = print
 LOGGING_INITIALIZED = False
@@ -1185,6 +1199,55 @@ class OutputMultiplexerContext(OutputMultiplexer, contextlib.ContextDecorator):
         return True
 
 
+def _timestamp() -> str:
+    ts = datetime.datetime.now(pytz.timezone('US/Pacific'))
+    return ts.strftime("%Y/%m/%dT%H:%M:%S.%f%z")
+
+
+def non_zero_return_value(ret: Any):
+    """
+    Special method hooked from bootstrap.py to optionally keep a system-wide
+    record of non-zero python program exits.
+
+    Args:
+        ret: the return value
+    """
+    try:
+        record = config.config['logging_non_zero_exits_record_path']
+        if record:
+            program = config.PROGRAM_NAME
+            args = config.ORIG_ARGV
+            with open(record, 'a') as af:
+                print(
+                    f'{_timestamp()}: {program} ({args}) exited with non-zero value {ret}.',
+                    file=af,
+                )
+    except Exception:
+        pass
+
+
+def unhandled_top_level_exception(exc_type: type):
+    """
+    Special method hooked from bootstrap.py to optionally keep a system-wide
+    record of unhandled top level exceptions.
+
+    Args:
+        exc_type: the type of the unhandled exception
+    """
+    try:
+        record = config.config['logging_unhandled_top_level_exceptions_record_path']
+        if record:
+            program = config.PROGRAM_NAME
+            args = config.ORIG_ARGV
+            with open(record, 'a') as af:
+                print(
+                    f'{_timestamp}: {program} ({args}) took unhandled top level exception {exc_type}',
+                    file=af,
+                )
+    except Exception:
+        pass
+
+
 def hlog(message: str) -> None:
     """Write a message to the house log (syslog facility local7 priority
     info) by calling `/usr/bin/logger`.  This is pretty hacky but used