X-Git-Url: https://wannabe.guru.org/gitweb/?a=blobdiff_plain;f=src%2Fpyutils%2Fparallelize%2Fparallelize.py;h=3cf1f6abc93305660b13a1c252bc8cf06c708a4c;hb=HEAD;hp=515d431dea4e4fa8c7c4101a0573d4308990c0e5;hpb=278d163705facc2276cd464414fb490ef6af50ab;p=pyutils.git

diff --git a/src/pyutils/parallelize/parallelize.py b/src/pyutils/parallelize/parallelize.py
index 515d431..3cf1f6a 100644
--- a/src/pyutils/parallelize/parallelize.py
+++ b/src/pyutils/parallelize/parallelize.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# © Copyright 2021-2022, Scott Gasch
+# © Copyright 2021-2023, Scott Gasch
 
 """A decorator to help with simple parallelization.  When decorated
 functions are invoked they execute on a background thread, process or
@@ -12,10 +12,12 @@ remote machine depending on the style of decoration::
     def my_function(a, b, c) -> int:
         ...do some slow / expensive work, e.g., an http request
 
+    # Run with background subprocess
     @p.parallelize(method=Method.PROCESS)
     def my_other_function(d, e, f) -> str:
         ...do more really expensive work, e.g., a network read
 
+    # Run in a helper process on another machine.
     @p.parallelize(method=Method.REMOTE)
     def my_other_other_function(g, h) -> int:
         ...this work will be distributed to a remote machine pool
@@ -72,6 +74,33 @@ do some setup work:
     If you're trying to set this up and struggling, email me at
     scott.gasch@gmail.com.  I'm happy to help.
 
+    What you get back when you call a decorated function (using
+    threads, processes or a remote worker) is a
+    :class:`pyutils.parallelize.smart_future.SmartFuture`.  This class
+    attempts to transparently wrap a normal Python :class:`Future`
+    (see
+    https://docs.python.org/3/library/concurrent.futures.html#future-objects).
+    If your code just uses the result of a `parallelized` method it
+    will block waiting on the result of the wrapped function as soon
+    as it uses that result in a manner that requires its value to be
+    known (e.g. using it in an expression, calling a method on it,
+    passing it into a method, hashing it / using it as a dict key,
+    etc...)  But you can do operations that do not require the value
+    to be known (e.g. storing it in a list, storing it as a value in a
+    dict, etc...) safely without blocking.
+
+    There are two helper methods in
+    :mod:`pyutils.parallelize.smart_future` to help deal with these
+    :class:`SmartFuture` objects called
+    :meth:`pyutils.parallelize.smart_future.wait_all` and
+    :meth:`pyutils.parallelize.smart_future.wait_any`.  These, when
+    given a collection of :class:`SmartFuture` objects,
+    will block until one (any) or all (all) are finished and yield the
+    finished objects to the caller.  Callers can be confident that any
+    objects returned from these methods will not block when accessed.
+    See documentation in :mod:`pyutils.parallelize.smart_future` for
+    more details.
+
 """
 
 import atexit
@@ -102,10 +131,12 @@ def parallelize(
         def my_function(a, b, c) -> int:
             ...do some slow / expensive work, e.g., an http request
 
+        # Run with background subprocess
         @p.parallelize(method=Method.PROCESS)
         def my_other_function(d, e, f) -> str:
             ...do more really expensive work, e.g., a network read
 
+        # Run in a helper process on another machine.
         @p.parallelize(method=Method.REMOTE)
         def my_other_other_function(g, h) -> int:
             ...this work will be distributed to a remote machine pool