Multiprocessing Manager Dict not updating across multiple processes

Question

I have a problem using the multiprocessing manager with dicts, to share values across all my processes. I am sure I have the basics right, but somehow I am just missing something…

Here is the Minimal repuducable example of my problem:

from multiprocessing import Process, Manager
import sys
import time
import random

def run_pipeline(position, frame):
    while True:
        position = {1: random.randint(0, 100)}
        frame = {2: random.randint(0, 100)}
        print("pipeline running")
        time.sleep(5)

def run_relay_controller(relay_status):
    while True:
        print("relay running")
        relay_status = {2: random.randint(0, 2)}
        time.sleep(5)

def run_webserver( frame, relay_status):
    while True:
        print("webserver running")
        frame = {1: random.randint(0, 100)}
        relay_status = {1: random.randint(0, 2)}
        time.sleep(5)

if __name__ == "__main__":
    manager = Manager()
    position = manager.dict()
    frame = manager.dict()
    relay_status = manager.dict()

    position_process = Process(target=run_pipeline, args=(position, frame))
    relay_process = Process(target=run_relay_controller, args=(relay_status, ))
    server_process = Process(target=run_webserver, args=( frame, relay_status))
    position_process.start()
    relay_process.start()
    server_process.start()

    while True:
        try:
            print(frame)
            print(relay_status)
            print(position)
        except KeyboardInterrupt:
            position_process.terminate()
            relay_process.terminate()
            server_process.terminate()
            break
    position_process.join()
    relay_process.join()
    server_process.join()
    sys.exit()

Asked By: Moritz Pfennig

||

Source

Answer 1

Corrected code using context manager and dictionary updates:

from multiprocessing import Process, Manager
import time
import random


def run_pipeline(position, frame):
    while True:
        position[1] = random.randint(0, 100)
        frame[2] = random.randint(0, 100)
        print("pipeline running")
        time.sleep(5)


def run_relay_controller(relay_status):
    while True:
        print("relay running")
        relay_status[2] = random.randint(0, 2)
        time.sleep(5)


def run_webserver(frame, relay_status):
    while True:
        print("webserver running")
        frame[1] = random.randint(0, 100)
        relay_status[1] = random.randint(0, 2)
        time.sleep(5)


if __name__ == "__main__":
    with Manager() as manager:
        position = manager.dict()
        frame = manager.dict()
        relay_status = manager.dict()

        position_process = Process(target=run_pipeline, args=(position, frame))
        relay_process = Process(target=run_relay_controller, args=(relay_status, ))
        server_process = Process(target=run_webserver, args=(frame, relay_status))
        position_process.start()
        relay_process.start()
        server_process.start()

        while True:
            try:
                print(frame)
                print(relay_status)
                print(position)
            except KeyboardInterrupt:
                position_process.terminate()
                relay_process.terminate()
                server_process.terminate()
                break
        position_process.join()
        relay_process.join()
        server_process.join()

Note:

Use of terminate() should be discouraged. Better to use some kind of flag (e.g., Manager.Event, Manager.Value). In this case, termination is triggered by KeyboardInterrupt which willl propagate to the sub-processes and can therefore be managed (within each subprocess) with try/except

Answered By: DarkKnight

Multiprocessing Manager Dict not updating across multiple processes

Question:

Answers: