Overriden `Process.run` does not execute asynchronously
Question:
Having subclassed Process.run
import multiprocessing as mp
import time
DELAY = 2
class NewProcess(mp.get_context().Process):
def run(self) -> None:
# add new kwarg to item[4] slot
old_que = self._args[0]
new_que = mp.SimpleQueue()
while not old_que.empty():
item = old_que.get()
new_que.put(
(
item[0],
item[1],
item[2], # Function
item[3], # Arguments
item[4] # Keyword arguments
| {
"message": "Hello world!",
},
)
)
# Recreate args
self._args = new_que, *self._args[1:]
# Continue as normal
super().run()
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
context = mp.get_context()
context.Process = NewProcess
with context.Pool(2) as pool:
responses = []
start = time.perf_counter()
for _ in range(2):
resp = pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
responses.append(resp)
for resp in responses:
resp.wait()
responses = [resp.get() for resp in responses]
total = time.perf_counter() - start
assert total - DELAY < 1e-2, f"Expected to take circa {DELAY}s, took {total}s"
assert responses == (
expected := list(
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
)
), f"{responses=}!={expected}"
I would expect that delay
function executes asynchronously taking circa DELAY
time. However, it does not. Script fails with
Traceback (most recent call last):
File "/home/vahvero/Desktop/tmp.py", line 54, in <module>
assert total - DELAY < 1e-2, f"Expected to take circa {DELAY}s, took {total}s"
AssertionError: Expected to take circa 2s, took 4.003754430001209s
Why my changes to run
cause linear rather than parallel processing?
Answers:
You have what I believe to be 3 issues with your code, two minor and one major:
- What you state should be the expected
responses
values is not correct, i.e. you should be expecting two (2) tuples in the returned list.
- The time you expect the two tasks to run needs to be a bit more generous as on a slower machine the overhead of creating and initializing processes could very well be more than .01 secods.
- Each pool process is creating its own new
SimpleQueue
instance. So instead of having a single queue instance with two items on it, we now have 2 queue instances each with two items on it for a total of 4 tasks to process. This is why it is taking twice as long as you think it should.
The following code does not create any new queue instances but instead modifies the items (tasks) on the queue if necessary. I ensure that the rewriting of the queue is performed only once. The code also prints out the results and timing:
import multiprocessing as mp
import time
from functools import partial
DELAY = 2
class NewProcess(mp.get_context().Process):
def __init__(self, rewritten, *args, **kwargs):
self._rewritten = rewritten
super().__init__(*args, **kwargs)
def run(self) -> None:
que = self._args[0]
# add new kwarg to item[4] slot
with self._rewritten.get_lock():
# has the queue been rewritten?
if not self._rewritten.value:
items = []
while not que.empty():
item = que.get()
if not item[4]:
item = list(item)
item[4] = {"message": "Hello world!"}
item = tuple(item)
items.append(item)
for item in items:
que.put(item)
self._rewritten.value = 1
super().run()
def create_new_process(rewritten, *args, **kwargs):
return NewProcess(rewritten, *args, **kwargs)
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
context = mp.get_context()
rewritten = mp.Value('i', 0) # Queue has not yet been rewritten
context.Process = partial(create_new_process, rewritten)
with context.Pool(2) as pool:
responses = []
start = time.time()
for _ in range(2):
resp = pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
responses.append(resp)
"""
# This is unnecessary: # Booboo
for resp in responses:
resp.wait()
"""
responses = [resp.get() for resp in responses]
total = time.time() - start
print(responses)
print(total)
# Be a bit more time tolerant: # Booboo
assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"
# You are expecting 2 items returned:
assert responses == (
expected := [
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
] * 2 # Note this line
), f"{responses=}!={expected}"
Prints:
[((0, 1, 2), {'message': 'Hello world!'}), ((0, 1, 2), {'message': 'Hello world!'})]
2.1168272495269775
Note
The reason why I mentioned in my comment the XY problem is because I wanted to understand what your ultimate goal was and to determine whether there was a better, safer way of accomplishing this. Your goal seems to be to ensure that if no keyword arguments were passed to your worker function, then you would provide a default. If so, surely there is a cleaner, simpler, more efficient way. For example, we can use a decorator function:
import multiprocessing as mp
import time
from functools import wraps
DELAY = 2
def provide_default_keywords(f):
@wraps(f)
def wrapper(*args, **kwargs):
if not kwargs:
kwargs = {"message": "Hello world!"}
return f(*args, **kwargs)
return wrapper
@provide_default_keywords
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
with mp.Pool(2) as pool:
responses = []
start = time.time()
for _ in range(2):
resp = pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
responses.append(resp)
responses = [resp.get() for resp in responses]
total = time.time() - start
print(responses)
print(total)
# Be a bit more time tolerant: # Booboo
assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"
# You are expecting 2 items returned:
assert responses == (
expected := [
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
] * 2 # Note this line
), f"{responses=}!={expected}"
Update
See my comment below concerning the race condition that renders your initial approach problematic. If you don’t want to use a decorator, then better would be to override the apply_async
method. In the following code I create a mixin class to do just that and it can be used with a multiprocessing pool or a multithreading pool:
from multiprocessing.pool import Pool
import time
DELAY = 2
class Apply_Async_Mixin: # Can be used with multiprocessing or multithreading
def apply_async(self, func, args=(), kwds={}, callback=None,
error_callback=None):
if not kwds:
kwds = {"message": "Hello world!"}
return super().apply_async(func,
args=args,
kwds=kwds,
callback=callback,
error_callback=error_callback)
# You must specify the mixin first:
class MyPool(Apply_Async_Mixin, Pool): # multiprocessing
pass
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
with MyPool(2) as pool:
start = time.time()
async_results = [
pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
for _ in range(2)
]
responses = [async_result.get() for async_result in async_results]
total = time.time() - start
print(responses)
print(total)
# Be a bit more time tolerant: # Booboo
assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"
# You are expecting 2 items returned:
assert responses == (
expected := [
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
] * 2 # Note this line
), f"{responses=}!={expected}"
Having subclassed Process.run
import multiprocessing as mp
import time
DELAY = 2
class NewProcess(mp.get_context().Process):
def run(self) -> None:
# add new kwarg to item[4] slot
old_que = self._args[0]
new_que = mp.SimpleQueue()
while not old_que.empty():
item = old_que.get()
new_que.put(
(
item[0],
item[1],
item[2], # Function
item[3], # Arguments
item[4] # Keyword arguments
| {
"message": "Hello world!",
},
)
)
# Recreate args
self._args = new_que, *self._args[1:]
# Continue as normal
super().run()
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
context = mp.get_context()
context.Process = NewProcess
with context.Pool(2) as pool:
responses = []
start = time.perf_counter()
for _ in range(2):
resp = pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
responses.append(resp)
for resp in responses:
resp.wait()
responses = [resp.get() for resp in responses]
total = time.perf_counter() - start
assert total - DELAY < 1e-2, f"Expected to take circa {DELAY}s, took {total}s"
assert responses == (
expected := list(
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
)
), f"{responses=}!={expected}"
I would expect that delay
function executes asynchronously taking circa DELAY
time. However, it does not. Script fails with
Traceback (most recent call last):
File "/home/vahvero/Desktop/tmp.py", line 54, in <module>
assert total - DELAY < 1e-2, f"Expected to take circa {DELAY}s, took {total}s"
AssertionError: Expected to take circa 2s, took 4.003754430001209s
Why my changes to run
cause linear rather than parallel processing?
You have what I believe to be 3 issues with your code, two minor and one major:
- What you state should be the expected
responses
values is not correct, i.e. you should be expecting two (2) tuples in the returned list. - The time you expect the two tasks to run needs to be a bit more generous as on a slower machine the overhead of creating and initializing processes could very well be more than .01 secods.
- Each pool process is creating its own new
SimpleQueue
instance. So instead of having a single queue instance with two items on it, we now have 2 queue instances each with two items on it for a total of 4 tasks to process. This is why it is taking twice as long as you think it should.
The following code does not create any new queue instances but instead modifies the items (tasks) on the queue if necessary. I ensure that the rewriting of the queue is performed only once. The code also prints out the results and timing:
import multiprocessing as mp
import time
from functools import partial
DELAY = 2
class NewProcess(mp.get_context().Process):
def __init__(self, rewritten, *args, **kwargs):
self._rewritten = rewritten
super().__init__(*args, **kwargs)
def run(self) -> None:
que = self._args[0]
# add new kwarg to item[4] slot
with self._rewritten.get_lock():
# has the queue been rewritten?
if not self._rewritten.value:
items = []
while not que.empty():
item = que.get()
if not item[4]:
item = list(item)
item[4] = {"message": "Hello world!"}
item = tuple(item)
items.append(item)
for item in items:
que.put(item)
self._rewritten.value = 1
super().run()
def create_new_process(rewritten, *args, **kwargs):
return NewProcess(rewritten, *args, **kwargs)
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
context = mp.get_context()
rewritten = mp.Value('i', 0) # Queue has not yet been rewritten
context.Process = partial(create_new_process, rewritten)
with context.Pool(2) as pool:
responses = []
start = time.time()
for _ in range(2):
resp = pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
responses.append(resp)
"""
# This is unnecessary: # Booboo
for resp in responses:
resp.wait()
"""
responses = [resp.get() for resp in responses]
total = time.time() - start
print(responses)
print(total)
# Be a bit more time tolerant: # Booboo
assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"
# You are expecting 2 items returned:
assert responses == (
expected := [
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
] * 2 # Note this line
), f"{responses=}!={expected}"
Prints:
[((0, 1, 2), {'message': 'Hello world!'}), ((0, 1, 2), {'message': 'Hello world!'})]
2.1168272495269775
Note
The reason why I mentioned in my comment the XY problem is because I wanted to understand what your ultimate goal was and to determine whether there was a better, safer way of accomplishing this. Your goal seems to be to ensure that if no keyword arguments were passed to your worker function, then you would provide a default. If so, surely there is a cleaner, simpler, more efficient way. For example, we can use a decorator function:
import multiprocessing as mp
import time
from functools import wraps
DELAY = 2
def provide_default_keywords(f):
@wraps(f)
def wrapper(*args, **kwargs):
if not kwargs:
kwargs = {"message": "Hello world!"}
return f(*args, **kwargs)
return wrapper
@provide_default_keywords
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
with mp.Pool(2) as pool:
responses = []
start = time.time()
for _ in range(2):
resp = pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
responses.append(resp)
responses = [resp.get() for resp in responses]
total = time.time() - start
print(responses)
print(total)
# Be a bit more time tolerant: # Booboo
assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"
# You are expecting 2 items returned:
assert responses == (
expected := [
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
] * 2 # Note this line
), f"{responses=}!={expected}"
Update
See my comment below concerning the race condition that renders your initial approach problematic. If you don’t want to use a decorator, then better would be to override the apply_async
method. In the following code I create a mixin class to do just that and it can be used with a multiprocessing pool or a multithreading pool:
from multiprocessing.pool import Pool
import time
DELAY = 2
class Apply_Async_Mixin: # Can be used with multiprocessing or multithreading
def apply_async(self, func, args=(), kwds={}, callback=None,
error_callback=None):
if not kwds:
kwds = {"message": "Hello world!"}
return super().apply_async(func,
args=args,
kwds=kwds,
callback=callback,
error_callback=error_callback)
# You must specify the mixin first:
class MyPool(Apply_Async_Mixin, Pool): # multiprocessing
pass
def delay(*args, **kwargs):
time.sleep(DELAY)
return args, kwargs
if __name__ == "__main__":
with MyPool(2) as pool:
start = time.time()
async_results = [
pool.apply_async(
func=delay,
args=tuple(range(3)),
kwds={},
)
for _ in range(2)
]
responses = [async_result.get() for async_result in async_results]
total = time.time() - start
print(responses)
print(total)
# Be a bit more time tolerant: # Booboo
assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"
# You are expecting 2 items returned:
assert responses == (
expected := [
(
(0, 1, 2),
{
"message": "Hello world!"
}
)
] * 2 # Note this line
), f"{responses=}!={expected}"