From: Scott <scott@wannabe.house>
Date: Wed, 2 Feb 2022 17:14:12 +0000 (-0800)
Subject: More type annotations.
X-Git-Url: https://wannabe.guru.org/gitweb/?a=commitdiff_plain;h=a4bf4d05230474ad14243d67ac7f8c938f670e58;p=python_utils.git

More type annotations.
---

diff --git a/arper.py b/arper.py
index 7700d5a..ca5d1d5 100644
--- a/arper.py
+++ b/arper.py
@@ -47,7 +47,7 @@ cfg.add_argument(
 )
 
 
-@persistent.persistent_autoloaded_singleton()
+@persistent.persistent_autoloaded_singleton()  # type: ignore
 class Arper(persistent.Persistent):
     def __init__(self, cached_state: Optional[BiDict] = None) -> None:
         self.state = BiDict()
diff --git a/base_presence.py b/base_presence.py
index 612193e..3ceddb3 100755
--- a/base_presence.py
+++ b/base_presence.py
@@ -4,7 +4,7 @@ import datetime
 from collections import defaultdict
 import logging
 import re
-from typing import Dict, List, Set
+from typing import Dict, List, Optional, Set
 import warnings
 
 # Note: this module is fairly early loaded.  Be aware of dependencies.
@@ -75,7 +75,7 @@ class PresenceDetection(object):
         ] = defaultdict(dict)
         self.names_by_mac: Dict[str, str] = {}
         self.dark_locations: Set[Location] = set()
-        self.last_update = None
+        self.last_update: Optional[datetime.datetime] = None
 
     def maybe_update(self) -> None:
         if self.last_update is None:
diff --git a/bootstrap.py b/bootstrap.py
index e50cb38..98da78c 100644
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -3,7 +3,9 @@
 import functools
 import logging
 import os
+import importlib
 from inspect import stack
+from typing import List
 import sys
 
 # This module is commonly used by others in here and should avoid
@@ -99,7 +101,7 @@ def handle_uncaught_exception(exc_type, exc_value, exc_tb):
                 original_hook(exc_type, exc_value, exc_tb)
 
 
-class ImportInterceptor(object):
+class ImportInterceptor(importlib.abc.MetaPathFinder):
     def __init__(self):
         import collect.trie
 
@@ -120,6 +122,11 @@ class ImportInterceptor(object):
     def should_ignore_filename(self, filename: str) -> bool:
         return 'importlib' in filename or 'six.py' in filename
 
+    def find_module(self, fullname, path):
+        raise Exception(
+            "This method has been deprecated since Python 3.4, please upgrade."
+        )
+
     def find_spec(self, loaded_module, path=None, target=None):
         s = stack()
         for x in range(3, len(s)):
@@ -147,6 +154,9 @@ class ImportInterceptor(object):
         logger.debug(msg)
         print(msg)
 
+    def invalidate_caches(self):
+        pass
+
     def find_importer(self, module: str):
         if module in self.tree_node_by_module:
             node = self.tree_node_by_module[module]
@@ -166,7 +176,7 @@ import_interceptor = None
 for arg in sys.argv:
     if arg == '--audit_import_events':
         import_interceptor = ImportInterceptor()
-        sys.meta_path = [import_interceptor] + sys.meta_path
+        sys.meta_path.insert(0, import_interceptor)
 
 
 def dump_all_objects() -> None:
diff --git a/decorator_utils.py b/decorator_utils.py
index a956a21..cd69639 100644
--- a/decorator_utils.py
+++ b/decorator_utils.py
@@ -223,7 +223,7 @@ def debug_count_calls(func: Callable) -> Callable:
         logger.info(msg)
         return func(*args, **kwargs)
 
-    wrapper_debug_count_calls.num_calls = 0
+    wrapper_debug_count_calls.num_calls = 0  # type: ignore
     return wrapper_debug_count_calls
 
 
@@ -366,7 +366,7 @@ def memoized(func: Callable) -> Callable:
             logger.debug(f"Returning memoized value for {func.__name__}")
         return wrapper_memoized.cache[cache_key]
 
-    wrapper_memoized.cache = dict()
+    wrapper_memoized.cache = dict()  # type: ignore
     return wrapper_memoized
 
 
diff --git a/dict_utils.py b/dict_utils.py
index b1464c6..451a87d 100644
--- a/dict_utils.py
+++ b/dict_utils.py
@@ -70,7 +70,7 @@ def raise_on_duplicated_keys(key, new_value, old_value):
 def coalesce(
     inputs: Iterator[Dict[Any, Any]],
     *,
-    aggregation_function: Callable[[Any, Any], Any] = coalesce_by_creating_list,
+    aggregation_function: Callable[[Any, Any, Any], Any] = coalesce_by_creating_list,
 ) -> Dict[Any, Any]:
     """Merge N dicts into one dict containing the union of all keys /
     values in the input dicts.  When keys collide, apply the
@@ -223,7 +223,7 @@ def dict_to_key_value_lists(d: Dict[Any, Any]) -> Tuple[List[Any], List[Any]]:
     ['scott', '555-1212', '123 main st.', '12345']
 
     """
-    r = ([], [])
+    r: Tuple[List[Any], List[Any]] = ([], [])
     for (k, v) in d.items():
         r[0].append(k)
         r[1].append(v)
diff --git a/executors.py b/executors.py
index e95ed71..990df03 100644
--- a/executors.py
+++ b/executors.py
@@ -248,7 +248,7 @@ class BundleDetails:
     end_ts: float
     slower_than_local_p95: bool
     slower_than_global_p95: bool
-    src_bundle: BundleDetails
+    src_bundle: Optional[BundleDetails]
     is_cancelled: threading.Event
     was_cancelled: bool
     backup_bundles: Optional[List[BundleDetails]]
@@ -288,7 +288,7 @@ class RemoteExecutorStatus:
         self.worker_count: int = total_worker_count
         self.known_workers: Set[RemoteWorkerRecord] = set()
         self.start_time: float = time.time()
-        self.start_per_bundle: Dict[str, float] = defaultdict(float)
+        self.start_per_bundle: Dict[str, Optional[float]] = defaultdict(float)
         self.end_per_bundle: Dict[str, float] = defaultdict(float)
         self.finished_bundle_timings_per_worker: Dict[
             RemoteWorkerRecord, List[float]
@@ -345,7 +345,9 @@ class RemoteExecutorStatus:
         self.end_per_bundle[uuid] = ts
         self.in_flight_bundles_by_worker[worker].remove(uuid)
         if not was_cancelled:
-            bundle_latency = ts - self.start_per_bundle[uuid]
+            start = self.start_per_bundle[uuid]
+            assert start
+            bundle_latency = ts - start
             x = self.finished_bundle_timings_per_worker.get(worker, list())
             x.append(bundle_latency)
             self.finished_bundle_timings_per_worker[worker] = x
@@ -836,9 +838,10 @@ class RemoteExecutor(BaseExecutor):
         return self.wait_for_process(p, bundle, 0)
 
     def wait_for_process(
-        self, p: subprocess.Popen, bundle: BundleDetails, depth: int
+        self, p: Optional[subprocess.Popen], bundle: BundleDetails, depth: int
     ) -> Any:
         machine = bundle.machine
+        assert p
         pid = p.pid
         if depth > 3:
             logger.error(
@@ -981,10 +984,12 @@ class RemoteExecutor(BaseExecutor):
 
             # Tell the original to stop if we finished first.
             if not was_cancelled:
+                orig_bundle = bundle.src_bundle
+                assert orig_bundle
                 logger.debug(
-                    f'{bundle}: Notifying original {bundle.src_bundle.uuid} we beat them to it.'
+                    f'{bundle}: Notifying original {orig_bundle.uuid} we beat them to it.'
                 )
-                bundle.src_bundle.is_cancelled.set()
+                orig_bundle.is_cancelled.set()
         self.release_worker(bundle, was_cancelled=was_cancelled)
         return result
 
@@ -1068,7 +1073,9 @@ class RemoteExecutor(BaseExecutor):
         # they will move the result_file to this machine and let
         # the original pick them up and unpickle them.
 
-    def emergency_retry_nasty_bundle(self, bundle: BundleDetails) -> fut.Future:
+    def emergency_retry_nasty_bundle(
+        self, bundle: BundleDetails
+    ) -> Optional[fut.Future]:
         is_original = bundle.src_bundle is None
         bundle.worker = None
         avoid_last_machine = bundle.machine
diff --git a/file_utils.py b/file_utils.py
index cd37f30..f273ea4 100644
--- a/file_utils.py
+++ b/file_utils.py
@@ -14,7 +14,7 @@ import time
 from typing import Optional
 import glob
 from os.path import isfile, join, exists
-from typing import List
+from typing import List, TextIO
 from uuid import uuid4
 
 
@@ -332,11 +332,13 @@ def get_file_md5(filename: str) -> str:
 
 def set_file_raw_atime(filename: str, atime: float):
     mtime = get_file_raw_mtime(filename)
+    assert mtime
     os.utime(filename, (atime, mtime))
 
 
 def set_file_raw_mtime(filename: str, mtime: float):
     atime = get_file_raw_atime(filename)
+    assert atime
     os.utime(filename, (atime, mtime))
 
 
@@ -434,8 +436,8 @@ def describe_file_mtime(filename: str, *, brief=False) -> Optional[str]:
     return describe_file_timestamp(filename, lambda x: x.st_mtime, brief=brief)
 
 
-def touch_file(filename: str, *, mode: Optional[int] = 0o666) -> bool:
-    return pathlib.Path(filename, mode=mode).touch()
+def touch_file(filename: str, *, mode: Optional[int] = 0o666):
+    pathlib.Path(filename, mode=mode).touch()
 
 
 def expand_globs(in_filename: str):
@@ -470,14 +472,14 @@ class FileWriter(object):
         self.filename = filename
         uuid = uuid4()
         self.tempfile = f'{filename}-{uuid}.tmp'
-        self.handle = None
+        self.handle: Optional[TextIO] = None
 
-    def __enter__(self) -> io.TextIOWrapper:
+    def __enter__(self) -> TextIO:
         assert not does_path_exist(self.tempfile)
         self.handle = open(self.tempfile, mode="w")
         return self.handle
 
-    def __exit__(self, exc_type, exc_val, exc_tb) -> bool:
+    def __exit__(self, exc_type, exc_val, exc_tb) -> Optional[bool]:
         if self.handle is not None:
             self.handle.close()
             cmd = f'/bin/mv -f {self.tempfile} {self.filename}'
diff --git a/histogram.py b/histogram.py
index 993b503..d45e93f 100644
--- a/histogram.py
+++ b/histogram.py
@@ -85,8 +85,8 @@ class SimpleHistogram(Generic[T]):
             return txt
 
         max_label_width: Optional[int] = None
-        lowest_start: int = None
-        highest_end: int = None
+        lowest_start: Optional[int] = None
+        highest_end: Optional[int] = None
         for bucket in sorted(self.buckets, key=lambda x: x[0]):
             start = bucket[0]
             if lowest_start is None:
diff --git a/persistent.py b/persistent.py
index d62dd67..7136559 100644
--- a/persistent.py
+++ b/persistent.py
@@ -64,6 +64,7 @@ def was_file_written_today(filename: str) -> bool:
         return False
 
     mtime = file_utils.get_file_mtime_as_datetime(filename)
+    assert mtime
     now = datetime.datetime.now()
     return mtime.month == now.month and mtime.day == now.day and mtime.year == now.year
 
@@ -80,6 +81,7 @@ def was_file_written_within_n_seconds(
         return False
 
     mtime = file_utils.get_file_mtime_as_datetime(filename)
+    assert mtime
     now = datetime.datetime.now()
     return (now - mtime).total_seconds() <= limit_seconds
 
@@ -126,7 +128,6 @@ class persistent_autoloaded_singleton(object):
         self.instance = None
 
     def __call__(self, cls: Persistent):
-        @functools.wraps(cls)
         def _load(*args, **kwargs):
 
             # If class has already been loaded, act like a singleton
diff --git a/smart_future.py b/smart_future.py
index 604c149..1f6e6f0 100644
--- a/smart_future.py
+++ b/smart_future.py
@@ -5,7 +5,7 @@ import concurrent
 import concurrent.futures as fut
 import logging
 import traceback
-from typing import Callable, List, TypeVar
+from typing import Callable, List, Set, TypeVar
 
 from overrides import overrides
 
@@ -27,11 +27,11 @@ def wait_any(
 ):
     real_futures = []
     smart_future_by_real_future = {}
-    completed_futures = set()
-    for f in futures:
-        assert type(f) == SmartFuture
-        real_futures.append(f.wrapped_future)
-        smart_future_by_real_future[f.wrapped_future] = f
+    completed_futures: Set[fut.Future] = set()
+    for x in futures:
+        assert type(x) == SmartFuture
+        real_futures.append(x.wrapped_future)
+        smart_future_by_real_future[x.wrapped_future] = x
 
     while len(completed_futures) != len(real_futures):
         newly_completed_futures = concurrent.futures.as_completed(real_futures)
@@ -59,9 +59,9 @@ def wait_all(
     log_exceptions: bool = True,
 ) -> None:
     real_futures = []
-    for f in futures:
-        assert type(f) == SmartFuture
-        real_futures.append(f.wrapped_future)
+    for x in futures:
+        assert type(x) == SmartFuture
+        real_futures.append(x.wrapped_future)
 
     (done, not_done) = concurrent.futures.wait(
         real_futures, timeout=None, return_when=concurrent.futures.ALL_COMPLETED
diff --git a/thread_utils.py b/thread_utils.py
index 6035b09..2216127 100644
--- a/thread_utils.py
+++ b/thread_utils.py
@@ -61,7 +61,7 @@ def is_current_thread_main_thread() -> bool:
 
 def background_thread(
     _funct: Optional[Callable],
-) -> Tuple[threading.Thread, threading.Event]:
+) -> Callable[..., Tuple[threading.Thread, threading.Event]]:
     """A function decorator to create a background thread.
 
     *** Please note: the decorated function must take an shutdown ***