Get a unique ID for worker in python multiprocessing pool

You can use multiprocessing.Queue to store the ids and then get the id at initialization of the pool process.

Advantages:

• You do not need to rely on internals.
• If your use case is to manage resources/ devices then you can put in the device number directly. This will also ensure that no device is used twice: If you have more processes in your pool than devices, the additional processes will block on queue.get() and will not perform any work (This won’t block your porgram, or at least it did not when I tested).

Disadvantages:

• You have additional communication overhead and spawning the pool
  processes takes a tiny bit longer: Without the sleep(1) in the
  example all work might be performed by the first process, as others
  are not done initializing, yet.
• You need a global (or at least I
  don’t know a way around it)

Example:

import multiprocessing
from time import sleep

def init(queue):
    global idx
    idx = queue.get()

def f(x):
    global idx
    process = multiprocessing.current_process()
    sleep(1)
    return (idx, process.pid, x * x)

ids = [0, 1, 2, 3]
manager = multiprocessing.Manager()
idQueue = manager.Queue()

for i in ids:
    idQueue.put(i)

p = multiprocessing.Pool(8, init, (idQueue,))
print(p.map(f, range(8)))

Output:

[(0, 8289, 0), (1, 8290, 1), (2, 8294, 4), (3, 8291, 9), (0, 8289, 16), (1, 8290, 25), (2, 8294, 36), (3, 8291, 49)]

Note, that there are only 4 different pid, although the pool contains 8 processes and one idx is only used by one process.

I did this with threading and ended up using a queue to handle job management. Here is the baseline. My complete version has a bunch of try-catches (particularly in the worker, to make sure that q.task_done() is called even on failure).

from threading import Thread
from queue import Queue
import time
import random


def run(idx, *args):
    time.sleep(random.random() * 1)
    print idx, ':', args


def run_jobs(jobs, workers=1):
    q = Queue()
    def worker(idx):
        while True:
            args = q.get()
            run(idx, *args)
            q.task_done()

    for job in jobs:
        q.put(job)

    for i in range(0, workers):
        t = Thread(target=worker, args=[i])
        t.daemon = True
        t.start()

    q.join()


if __name__ == "__main__":
    run_jobs([('job', i) for i in range(0,10)], workers=5)

I didn’t need to use multiprocessing (my workers are just for calling an external process), but this could be extended. The API for multiprocessing changes it a touch, here’s how you could adapt:

from multiprocessing import Process, Queue
from Queue import Empty
import time
import random

def run(idx, *args):
    time.sleep(random.random() * i)
    print idx, ':', args


def run_jobs(jobs, workers=1):
    q = Queue()
    def worker(idx):
        try:
            while True:
                args = q.get(timeout=1)
                run(idx, *args)
        except Empty:
            return

    for job in jobs:
        q.put(job)

    processes = []
    for i in range(0, workers):
        p = Process(target=worker, args=[i])
        p.daemon = True
        p.start()
        processes.append(p)

    for p in processes: 
        p.join()


if __name__ == "__main__":
    run_jobs([('job', i) for i in range(0,10)], workers=5)

Both versions will output something like:

0 : ('job', 0)
1 : ('job', 2)
1 : ('job', 6)
3 : ('job', 3)
0 : ('job', 5)
1 : ('job', 7)
2 : ('job', 1)
4 : ('job', 4)
3 : ('job', 8)
0 : ('job', 9)

I’m not sure how it would work with Pool, but printing Process gives some unique output:

x = Process(target=time.sleep, args=[20])
x.start()
print(x)  # <Process name='Process-5' pid=97121 parent=95732 started>

I managed to map to a class method by getting the function handle using getattr, then using a wrapper to pack and unpack as many arguments as I wanted to pass to the method being mapped. In my case I was passing methods from the same class where the pool was being launched, but you can also pass an object as well to map to different classes.

This is the code:

import multiprocessing
from multiprocessing import Pool


def warp(args):
    func = args[0]
    frame = args[1]
    left_over = args[2:]
    func(frame, *left_over)


class MyClass:

    def __init__(self):
        self.my_flag = 5

    def exec_method(self, method, int_list, *args):
        obj = getattr(self, method.__name__)

        packed = list()
        for i in int_list:
            pack = list()
            pack.append(obj)
            pack.append(i)
            for arg in args:
                pack.append(arg)
            packed.append(pack)

        print("Start")
        pool = Pool(processes=multiprocessing.cpu_count())
        pool.map(warp, packed)
        print("End")

    def method1(self, my_str):
        print(self.my_flag, my_str)

    def method2(self, i, print_str, bool_flat):
        print(multiprocessing.current_process(), self.my_flag, i, print_str, str(bool_flat))


cls: MyClass = MyClass()
cls.my_flag = 58
cls.exec_method(cls.method2, [1, 5, 10, 20, 30], "this is a string", True)

This is the output:

Start
<ForkProcess(ForkPoolWorker-1, started daemon)> 58 1 this is a string True
<ForkProcess(ForkPoolWorker-2, started daemon)> 58 5 this is a string True
<ForkProcess(ForkPoolWorker-4, started daemon)> 58 20 this is a string True
<ForkProcess(ForkPoolWorker-5, started daemon)> 58 30 this is a string True
<ForkProcess(ForkPoolWorker-3, started daemon)> 58 10 this is a string True
End

Here is another approach that could be considered :

import multiprocessing
import math

def worker(worker_id, pid_dict, data_dict, worker_dict, res_dict, nb_worker):
    current = multiprocessing.current_process()
    pid_dict[worker_id] = current.pid
    worker_dict[worker_id] = worker_id 
    data_len = len(data_dict)
    chunk_size = math.ceil(data_len / nb_worker)
    start_index = worker_id * chunk_size
    end_index = min(((worker_id + 1) * chunk_size - 1), data_len)
    res_dict[worker_id] = 0
    
    for i in range(start_index, end_index):
        res_dict[worker_id] = res_dict[worker_id] + data_dict[i]

if __name__ == "__main__":
    nb_worker = 7
    manager = multiprocessing.Manager()
    pid_dict = manager.dict()
    worker_dict = manager.dict()
    data_dict = manager.dict()
    res_dict = manager.dict()
        
    for i in range(100000):
        data_dict[i] = i
    
    jobs = []
    
    for i in range(nb_worker):
        p = multiprocessing.Process(target = worker, args = (i, pid_dict, data_dict, worker_dict, res_dict, nb_worker))
        jobs.append(p)
        p.start()

    for proc in jobs:
        proc.join()
        
    print("Pid")
    print(pid_dict.values())
    print(" ")
    print("Worker id")
    print(worker_dict.values())
    print(" ")
    print("Sum data by worker")
    print(res_dict.values())
    print(" ")