Make remote workers die if no longer needed; cleanups in executors.

[python_utils.git] / remote_worker.py

diff --git a/remote_worker.py b/remote_worker.py
index 84f8d56fa33318b507f3f11618c663861718182b..bf8de6c66a36767ac267cfdd2bffe38317cbace0 100755 (executable)
--- a/remote_worker.py
+++ b/remote_worker.py
@@ -6,6 +6,7 @@ results.
 
 import logging
 import os
+import platform
 import signal
 import threading
 import sys
@@ -50,16 +51,23 @@ cfg.add_argument(
 
 @background_thread
 def watch_for_cancel(terminate_event: threading.Event) -> None:
+    if platform.node() == 'VIDEO-COMPUTER':
+        logger.warning('Background thread not allowed on retarded computers, sorry.')
+        return
+    logger.debug('Starting up background thread...')
     p = psutil.Process(os.getpid())
     while True:
         saw_sshd = False
         ancestors = p.parents()
         for ancestor in ancestors:
             name = ancestor.name()
+            pid = ancestor.pid
+            logger.debug(f'Ancestor process {name} (pid={pid})')
             if 'ssh' in name.lower():
                 saw_sshd = True
                 break
         if not saw_sshd:
+            logger.error('Did not see sshd in our ancestors list?!  Committing suicide.')
             os.system('pstree')
             os.kill(os.getpid(), signal.SIGTERM)
             time.sleep(5.0)
@@ -75,6 +83,8 @@ def main() -> None:
     in_file = config.config['code_file']
     out_file = config.config['result_file']
 
+    (thread, stop_thread) = watch_for_cancel()
+
     logger.debug(f'Reading {in_file}.')
     try:
         with open(in_file, 'rb') as rb:
@@ -82,6 +92,7 @@ def main() -> None:
     except Exception as e:
         logger.exception(e)
         logger.critical(f'Problem reading {in_file}.  Aborting.')
+        stop_thread.set()
         sys.exit(-1)
 
     logger.debug(f'Deserializing {in_file}.')
@@ -90,6 +101,7 @@ def main() -> None:
     except Exception as e:
         logger.exception(e)
         logger.critical(f'Problem deserializing {in_file}.  Aborting.')
+        stop_thread.set()
         sys.exit(-1)
 
     logger.debug('Invoking user code...')
@@ -104,6 +116,7 @@ def main() -> None:
     except Exception as e:
         logger.exception(e)
         logger.critical(f'Could not serialize result ({type(ret)}).  Aborting.')
+        stop_thread.set()
         sys.exit(-1)
 
     logger.debug(f'Writing {out_file}.')
@@ -113,8 +126,12 @@ def main() -> None:
     except Exception as e:
         logger.exception(e)
         logger.critical(f'Error writing {out_file}.  Aborting.')
+        stop_thread.set()
         sys.exit(-1)
 
+    stop_thread.set()
+    thread.join()
+
 
 if __name__ == '__main__':
     main()