subprocess and shared self object in a call back

Question

I have a typical multiple producers with a single consumer using python subprocess and one queue. The consumer has a callback to another object. Although the object is shared with all the subprocesses, when the subprocesses are finished, the changes to the object are lost.

Here's the code:

from multiprocessing import Process, Queue
import random
import time

class Manager(object):
    def __init__(self):
        self.queue = Queue()

    def consume(self, call_back):
        while True:
            task = self.queue.get()
            if task is None:
                self.queue.close()
                break
            time.sleep(0.05)
            call_back(task)
            print("Queue got task: {}.".format(task))

    def produce(self, value):
        time.sleep(random.uniform(0.1, 1.0))
        task = "TSK {}".format(value)
        self.queue.put(task)


    def start(self, call_back, n_tasks=10):

        consumer = Process(target=self.consume, args=(call_back,))
        consumer.start()

        workers = [Process(target=self.produce,args=(i,))
            for i in range(n_tasks)]

        for w in workers:
            w.start()
        for w in workers:
            w.join()

        self.queue.put(None)
        consumer.join()


class Display(object):
    def __init__(self):
        self.tasks = []

    def display_tasks(self, n_tasks=10):
        def add_task(task):
            self.tasks.append(task)
            print("Dislaying tasks so far: {}".format(self.tasks))
        Manager().start(add_task, n_tasks)
        print("Total tasks: {}".format(self.tasks))


Display().display_tasks(5)

The output is:

Dislaying tasks so far: ['TSK 3']
Queue got task: TSK 3.
Dislaying tasks so far: ['TSK 3', 'TSK 4']
Queue got task: TSK 4.
Dislaying tasks so far: ['TSK 3', 'TSK 4', 'TSK 2']
Queue got task: TSK 2.
Dislaying tasks so far: ['TSK 3', 'TSK 4', 'TSK 2', 'TSK 0']
Queue got task: TSK 0.
Dislaying tasks so far: ['TSK 3', 'TSK 4', 'TSK 2', 'TSK 0', 'TSK 1']
Queue got task: TSK 1.
Total tasks: []

And I was expecting:

Total tasks: ['TSK 3', 'TSK 4', 'TSK 2', 'TSK 0', 'TSK 1']

Any idea how this can be accomplished without:

class Display(object):
    def __init__(self):
        manager = Manager()
        self.tasks = manager.list()

In fact in the real case this is using QTableWidget and every entry in the table is a QTableWidgetItem....

Here it is the real call back function (data_set_table is a QTableWidget):

def _add_item(data):
        row = dc.size()
        self._content.data_set_table.insertRow(row)
        for i in range(len(data)):
            if data[i] is not None:
                item = QtGui.QTableWidgetItem(str(data[i]))
                item.setFlags( QtCore.Qt.ItemIsSelectable |  QtCore.Qt.ItemIsEnabled )
                self._content.data_set_table.setItem(row, i, item)

score 0 · Answer 1 · answered Aug 26 '17 at 15:46

The answer appears to be using a Manager. I will delete the answer in it does not work qith PyQt:

from multiprocessing import Process, Queue, Manager from multiprocessing.managers import BaseManager
import random import time

class Scheduler(object):
    def __init__(self):
        self.queue = Queue()

    def consume(self, call_back):
        while True:
            task = self.queue.get()
            if task is None:
                self.queue.close()
                break
            time.sleep(0.05)
            print("Queue got task: {}.".format(task))
            call_back(task)

    def produce(self, value):
        time.sleep(random.uniform(0.1, 1.0))
        task = "TSK {}".format(value)
        self.queue.put(task)


    def start(self, call_back, n_tasks=10):

        consumer = Process(target=self.consume, args=(call_back,))
        consumer.start()

        workers = [Process(target=self.produce,args=(i,))
            for i in range(n_tasks)]

        for w in workers:
            w.start()
        for w in workers:
            w.join()

        self.queue.put(None)
        consumer.join()

class CustomTable(object):
    def __init__(self):
        self.contents = []

    def addItem(self, item):
        self.contents.append(item)

    def display(self):
        print(self.__repr__)

    def __repr__(self):
        res = 80*"-" + "\n" + str(self.contents) + "\n" + 80*"-" + "\n"
        return res

class CustomTableManager(BaseManager):  
    pass  

CustomTableManager.register('CustomTable', CustomTable, exposed = ['contents', 'addItem', 'display', '__repr__'])  

customTableManager = CustomTableManager()  
customTableManager.start()  

class Display(object):
    def __init__(self):
        # self.tasks = CustomTable()
        self.tasks = customTableManager.CustomTable()

    def display_tasks(self, n_tasks=10):
        def add_task(task):
            self.tasks.addItem(task)
            print("Dislaying tasks so far:\n{}".format(self.tasks))
        Scheduler().start(add_task, n_tasks)
        print("Total tasks:\n{}".format(self.tasks))

Display().display_tasks(5)

subprocess and shared self object in a call back

1 Answers1