X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=parallelize.py;h=5f1de2b5b5fdc043b32779c7126a283ac831c5da;hb=772038b8d852446b77c9ee857885874f2552e9a6;hp=98f883c5716ae8fa3d67101b1d38d20dece7a0b6;hpb=36fea7f15ed17150691b5b3ead75450e575229ef;p=python_utils.git

diff --git a/parallelize.py b/parallelize.py
index 98f883c..5f1de2b 100644
--- a/parallelize.py
+++ b/parallelize.py
@@ -1,25 +1,31 @@
 #!/usr/bin/env python3
 
+# © Copyright 2021-2022, Scott Gasch
+
 """A decorator to help with dead simple parallelization."""
 
+
 import atexit
-from enum import Enum
 import functools
 import typing
+from enum import Enum
 
 
 class Method(Enum):
+    """How should we parallelize; by threads, processes or remote workers?"""
+
     THREAD = 1
     PROCESS = 2
     REMOTE = 3
 
 
 def parallelize(
-    _funct: typing.Optional[typing.Callable] = None,
-    *,
-    method: Method = Method.THREAD
+    _funct: typing.Optional[typing.Callable] = None, *, method: Method = Method.THREAD
 ) -> typing.Callable:
-    """Usage:
+    """This is a decorator that was created to make multi-threading,
+    multi-processing and remote machine parallelism simple in python.
+
+    Sample usage::
 
         @parallelize    # defaults to thread-mode
         def my_function(a, b, c) -> int:
@@ -27,37 +33,39 @@ def parallelize(
 
         @parallelize(method=Method.PROCESS)
         def my_other_function(d, e, f) -> str:
-            ...do more really expensice work, e.g., a network read
+            ...do more really expensive work, e.g., a network read
 
         @parallelize(method=Method.REMOTE)
         def my_other_other_function(g, h) -> int:
             ...this work will be distributed to a remote machine pool
 
-    This decorator will invoke the wrapped function on:
+    This decorator will invoke the wrapped function on::
 
         Method.THREAD (default): a background thread
         Method.PROCESS: a background process
         Method.REMOTE: a process on a remote host
 
     The wrapped function returns immediately with a value that is
-    wrapped in a SmartFuture.  This value will block if it is either
-    read directly (via a call to result._resolve) or indirectly (by
-    using the result in an expression, printing it, hashing it,
-    passing it a function argument, etc...).  See comments on the
-    SmartFuture class for details.
-
-    Note: you may stack @parallelized methods and it will "work".
-    That said, having multiple layers of Method.PROCESS or
-    Method.REMOTE may prove to be problematic because each process in
-    the stack will use its own independent pool which may overload
-    your machine with processes or your network with remote processes
-    beyond the control mechanisms built into one instance of the pool.
-    Be careful.
-
-    Also note: there is a non trivial overhead of pickling code and
-    scp'ing it over the network when you use Method.REMOTE.  There's
-    a smaller but still considerable cost of creating a new process
-    and passing code to/from it when you use Method.PROCESS.
+    wrapped in a :class:`SmartFuture`.  This value will block if it is
+    either read directly (via a call to :meth:`_resolve`) or indirectly
+    (by using the result in an expression, printing it, hashing it,
+    passing it a function argument, etc...).  See comments on
+    :class:`SmartFuture` for details.
+
+    .. warning::
+        You may stack @parallelized methods and it will "work".
+        That said, having multiple layers of :code:`Method.PROCESS` or
+        :code:`Method.REMOTE` will prove to be problematic because each process in
+        the stack will use its own independent pool which may overload
+        your machine with processes or your network with remote processes
+        beyond the control mechanisms built into one instance of the pool.
+        Be careful.
+
+    .. note::
+        There is non-trivial overhead of pickling code and
+        copying it over the network when you use :code:`Method.REMOTE`.  There's
+        a smaller but still considerable cost of creating a new process
+        and passing code to/from it when you use :code:`Method.PROCESS`.
     """
 
     def wrapper(funct: typing.Callable):