Overriden `Process.run` does not execute asynchronously

Question:

Having subclassed Process.run

import multiprocessing as mp
import time

DELAY = 2

class NewProcess(mp.get_context().Process):
    def run(self) -> None:
        # add new kwarg to item[4] slot
        old_que = self._args[0]
        new_que = mp.SimpleQueue()
        while not old_que.empty():
            item = old_que.get()
            new_que.put(
                (
                    item[0],
                    item[1],
                    item[2], # Function
                    item[3], # Arguments
                    item[4]  # Keyword arguments
                    | {
                        "message": "Hello world!",
                    },
                )
            )
        # Recreate args
        self._args = new_que, *self._args[1:]
        # Continue as normal
        super().run()


def delay(*args, **kwargs):
    time.sleep(DELAY)
    return args, kwargs


if __name__ == "__main__":
    context = mp.get_context()
    context.Process = NewProcess
    with context.Pool(2) as pool:
        responses = []
        start = time.perf_counter()
        for _ in range(2):
            resp = pool.apply_async(
                func=delay,
                args=tuple(range(3)),
                kwds={},
            )

            responses.append(resp)
        for resp in responses:
            resp.wait()

        responses = [resp.get() for resp in  responses]
        total = time.perf_counter() - start

        assert total - DELAY < 1e-2, f"Expected to take circa {DELAY}s, took {total}s"

        assert responses == (
            expected := list(
                (
                    (0, 1, 2),
                    {
                        "message": "Hello world!"
                    }
                )
            )
        ), f"{responses=}!={expected}"

I would expect that delay function executes asynchronously taking circa DELAY time. However, it does not. Script fails with

Traceback (most recent call last):
  File "/home/vahvero/Desktop/tmp.py", line 54, in <module>
    assert total - DELAY < 1e-2, f"Expected to take circa {DELAY}s, took {total}s"
AssertionError: Expected to take circa 2s, took 4.003754430001209s

Why my changes to run cause linear rather than parallel processing?

Asked By: vahvero

||

Answers:

You have what I believe to be 3 issues with your code, two minor and one major:

  1. What you state should be the expected responses values is not correct, i.e. you should be expecting two (2) tuples in the returned list.
  2. The time you expect the two tasks to run needs to be a bit more generous as on a slower machine the overhead of creating and initializing processes could very well be more than .01 secods.
  3. Each pool process is creating its own new SimpleQueue instance. So instead of having a single queue instance with two items on it, we now have 2 queue instances each with two items on it for a total of 4 tasks to process. This is why it is taking twice as long as you think it should.

The following code does not create any new queue instances but instead modifies the items (tasks) on the queue if necessary. I ensure that the rewriting of the queue is performed only once. The code also prints out the results and timing:

import multiprocessing as mp
import time
from functools import partial

DELAY = 2


class NewProcess(mp.get_context().Process):
    def __init__(self, rewritten, *args, **kwargs):
        self._rewritten = rewritten
        super().__init__(*args, **kwargs)

    def run(self) -> None:
        que = self._args[0]
        # add new kwarg to item[4] slot
        with self._rewritten.get_lock():
            # has the queue been rewritten?
            if not self._rewritten.value:
                items = []
                while not que.empty():
                    item = que.get()
                    if not item[4]:
                        item = list(item)
                        item[4] = {"message": "Hello world!"}
                        item = tuple(item)
                    items.append(item)
                for item in items:
                    que.put(item)
                self._rewritten.value = 1
        super().run()

def create_new_process(rewritten, *args, **kwargs):
    return NewProcess(rewritten, *args, **kwargs)

def delay(*args, **kwargs):
    time.sleep(DELAY)
    return args, kwargs


if __name__ == "__main__":
    context = mp.get_context()
    rewritten = mp.Value('i', 0) # Queue has not yet been rewritten
    context.Process = partial(create_new_process, rewritten)
    with context.Pool(2) as pool:
        responses = []
        start = time.time()
        for _ in range(2):
            resp = pool.apply_async(
                func=delay,
                args=tuple(range(3)),
                kwds={},
            )

            responses.append(resp)
        """
        # This is unnecessary: # Booboo
        for resp in responses:
            resp.wait()
        """

        responses = [resp.get() for resp in  responses]
        total = time.time() - start
        print(responses)
        print(total)

        # Be a bit more time tolerant: # Booboo
        assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"

        # You are expecting 2 items returned:
        assert responses == (
            expected := [
                (
                    (0, 1, 2),
                    {
                        "message": "Hello world!"
                    }
                )
            ] * 2 # Note this line
        ), f"{responses=}!={expected}"

Prints:

[((0, 1, 2), {'message': 'Hello world!'}), ((0, 1, 2), {'message': 'Hello world!'})]
2.1168272495269775

Note

The reason why I mentioned in my comment the XY problem is because I wanted to understand what your ultimate goal was and to determine whether there was a better, safer way of accomplishing this. Your goal seems to be to ensure that if no keyword arguments were passed to your worker function, then you would provide a default. If so, surely there is a cleaner, simpler, more efficient way. For example, we can use a decorator function:

import multiprocessing as mp
import time
from functools import wraps

DELAY = 2


def provide_default_keywords(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        if not kwargs:
            kwargs = {"message": "Hello world!"}
        return f(*args, **kwargs)
    return wrapper

@provide_default_keywords
def delay(*args, **kwargs):
    time.sleep(DELAY)
    return args, kwargs


if __name__ == "__main__":
    with mp.Pool(2) as pool:
        responses = []
        start = time.time()
        for _ in range(2):
            resp = pool.apply_async(
                func=delay,
                args=tuple(range(3)),
                kwds={},
            )

            responses.append(resp)

        responses = [resp.get() for resp in  responses]
        total = time.time() - start
        print(responses)
        print(total)

        # Be a bit more time tolerant: # Booboo
        assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"

        # You are expecting 2 items returned:
        assert responses == (
            expected := [
                (
                    (0, 1, 2),
                    {
                        "message": "Hello world!"
                    }
                )
            ] * 2 # Note this line
        ), f"{responses=}!={expected}"

Update

See my comment below concerning the race condition that renders your initial approach problematic. If you don’t want to use a decorator, then better would be to override the apply_async method. In the following code I create a mixin class to do just that and it can be used with a multiprocessing pool or a multithreading pool:

from multiprocessing.pool import Pool
import time

DELAY = 2

class Apply_Async_Mixin: # Can be used with multiprocessing or multithreading
    def apply_async(self, func, args=(), kwds={}, callback=None,
            error_callback=None):
        if not kwds:
            kwds = {"message": "Hello world!"}
        return super().apply_async(func,
                                   args=args,
                                   kwds=kwds,
                                   callback=callback,
                                   error_callback=error_callback)

# You must specify the mixin first:
class MyPool(Apply_Async_Mixin, Pool): # multiprocessing
    pass

def delay(*args, **kwargs):
    time.sleep(DELAY)
    return args, kwargs

if __name__ == "__main__":
    with MyPool(2) as pool:
        start = time.time()
        async_results = [
            pool.apply_async(
                func=delay,
                args=tuple(range(3)),
                kwds={},
            )
            for _ in range(2)
        ]
        responses = [async_result.get() for async_result in async_results]
        total = time.time() - start

        print(responses)
        print(total)

        # Be a bit more time tolerant: # Booboo
        assert total < DELAY + .2, f"Expected to take circa {DELAY}s, took {total}s"

        # You are expecting 2 items returned:
        assert responses == (
            expected := [
                (
                    (0, 1, 2),
                    {
                        "message": "Hello world!"
                    }
                )
            ] * 2 # Note this line
        ), f"{responses=}!={expected}"
Answered By: Booboo
Categories: questions Tags: ,
Answers are sorted by their score. The answer accepted by the question owner as the best is marked with
at the top-right corner.