Using multiprocessing and ProcessPoolExecutor simultaneously

Question

I am trying to create a simple script for python3.5 that can execute heavy computer vision algorithms in parallel. I have created a process by multiprocessing.Process in main process. Inside that process I create concurrent.futures.ProcessPoolExecutor. Spawned process submits tasks to processPoolExecutor and it works perfectly fine. But when I try to stop and join spawned process it hangs on join.

Also if replace processPoolExecuter to threadPoolExecuter everything works perfectly. What did I miss?

Here is main file:

import multiprocessing as mp
import queue as Queue
import numpy as np
import cv2
from time import sleep
import executer_debug

def worker(queue):
    pExecutor = executer_debug.Worker()
    pExecutor.set()

    while True:
        print("-->{}<--".format(pExecutor.get()))
        sleep(1)

        try:
            income = queue.get_nowait()
            break
        except Queue.Empty:
            pass
        pExecutor.set()

    print("<1>{}<1>".format(pExecutor.get()))
    print("<2>{}<2>".format(pExecutor.get()))

def main():

    queue = mp.Queue()
    currProcess = mp.Process(target = worker, args=(queue,))
    currProcess.start()

    frame = np.zeros((480,640), dtype=np.uint8)

    while True:
        cv2.imshow('frame',frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    print("stopped")
    queue.put("stop")

    currProcess.join()

if __name__ == "__main__":
    main()

And here is the second file. Code is very simple just enough to demonstrate the issue.

import collections
from concurrent.futures import ProcessPoolExecutor
from time import sleep
import multiprocessing as mp


def worker():
    return 1

class Worker():
    def __init__(self):
        self.workers_count = 4
        self.poolExecutor = ProcessPoolExecutor(max_workers = self.workers_count)
        self.executors = collections.deque()

    def set(self):
        self.executors.append(self.poolExecutor.submit(worker))

    def get(self):
        if len(self.executors) > 0:
            if self.executors[0].done():
                return self.executors.popleft().result()
            else:
                return 0
        else:
            return -1

Thank you!

Using multiprocessing and ProcessPoolExecutor simultaneously

0 Answers0