- _funct: typing.Optional[typing.Callable] = None,
- *,
- method: Method = Method.THREAD
+ _funct: typing.Optional[typing.Callable] = None, *, method: Method = Method.THREAD
- @parallelize # defaults to thread-mode
- def my_function(a, b, c) -> int:
- ...do some slow / expensive work, e.g., an http request
+ @parallelize # defaults to thread-mode
+ def my_function(a, b, c) -> int:
+ ...do some slow / expensive work, e.g., an http request
- @parallelize(method=Method.PROCESS)
- def my_other_function(d, e, f) -> str:
- ...do more really expensice work, e.g., a network read
+ @parallelize(method=Method.PROCESS)
+ def my_other_function(d, e, f) -> str:
+ ...do more really expensice work, e.g., a network read
- @parallelize(method=Method.REMOTE)
- def my_other_other_function(g, h) -> int:
- ...this work will be distributed to a remote machine pool
+ @parallelize(method=Method.REMOTE)
+ def my_other_other_function(g, h) -> int:
+ ...this work will be distributed to a remote machine pool
your machine with processes or your network with remote processes
beyond the control mechanisms built into one instance of the pool.
Be careful.
your machine with processes or your network with remote processes
beyond the control mechanisms built into one instance of the pool.
Be careful.
+
+ Also note: there is a non trivial overhead of pickling code and
+ scp'ing it over the network when you use Method.REMOTE. There's
+ a smaller but still considerable cost of creating a new process
+ and passing code to/from it when you use Method.PROCESS.
elif method == Method.REMOTE:
executor = executors.DefaultExecutors().remote_pool()
assert executor is not None
elif method == Method.REMOTE:
executor = executors.DefaultExecutors().remote_pool()
assert executor is not None