From 2a1a7b2a6beeca2f7fd11e6efd5b8ef7216ad5d7 Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Sun, 30 Oct 2022 22:53:53 -0700 Subject: [PATCH] Update docs in parallelize again. --- src/pyutils/parallelize/parallelize.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/pyutils/parallelize/parallelize.py b/src/pyutils/parallelize/parallelize.py index 7071ef3..0004f17 100644 --- a/src/pyutils/parallelize/parallelize.py +++ b/src/pyutils/parallelize/parallelize.py @@ -74,18 +74,20 @@ do some setup work: If you're trying to set this up and struggling, email me at scott.gasch@gmail.com. I'm happy to help. - What you get back when you call a decorated message is a + What you get back when you call a decorated function (using + threads, processes or a remote worker) is a :class:`pyutils.parallelize.smart_future.SmartFuture`. This class attempts to transparently wrap a normal Python :class:`Future` (see https://docs.python.org/3/library/concurrent.futures.html#future-objects). If your code just uses the result of a `parallelized` method it - will block waiting on the result of the method as soon as it uses - the result in a manner that requires its value to be known - (e.g. using it in an expression, calling a method on it, hashing - it, etc...) But you can do operations that do not require the - value to be known (e.g. storing it in a list, storing it as a - value in a dict, etc...) without blocking. + will block waiting on the result of the wrapped function as soon + as it uses that result in a manner that requires its value to be + known (e.g. using it in an expression, calling a method on it, + passing it into a method, hashing it / using it as a dict key, + etc...) But you can do operations that do not require the value + to be known (e.g. storing it in a list, storing it as a value in a + dict, etc...) safely without blocking. There are two helper methods in :mod:`pyutils.parallelize.smart_future` to help deal with these @@ -129,10 +131,12 @@ def parallelize( def my_function(a, b, c) -> int: ...do some slow / expensive work, e.g., an http request + # Run with background subprocess @p.parallelize(method=Method.PROCESS) def my_other_function(d, e, f) -> str: ...do more really expensive work, e.g., a network read + # Run in a helper process on another machine. @p.parallelize(method=Method.REMOTE) def my_other_other_function(g, h) -> int: ...this work will be distributed to a remote machine pool -- 2.45.2