I'm using multiprocessing's Pool().starmap()
function to run a type of cross-product computation, which is symmetrical, and i'm doing the same computation twice. How can i fix that to launch only half the processes ? What is taking me the more time is launching the processes and pickling the objects.
1° I have a list of objects X
, all of the same class, with a common method prod_scalar
such that X[i].prod_scalar(X[j]) == X[j].prod_scalar(X[i])
So i'm using code that looks like that :
import itertools as it
import multiprocessing as mp
import os
import numpy as np
class ComplicatedThing:
def __init__(self,number):
self.n = number
def interaction(self,other_ComplicatedThing):
# Do very complicated things to return a scalar result, which is symmetrical : x.interaction(y) == y.interaction(x)
return self.n*other_ComplicatedThing.n
def interact_with_myself(self):
# x.interact(x) is equal to x.interact_with_myself, but x.interact_with_myself() is a lot faster.
return self.n**2
def interact_with(obj1,obj2):
return obj1.interaction(obj2)
class MyClass:
def __init__(self,max):
self.objects = [ComplicatedThing(i) for i in range(max)]
pool = mp.Pool(os.cpu_count() -1)
self.result_matrix = np.array(pool.starmap(interact_with,it.product(self.objects,self.objects))).reshape(max,max)
if __name__ == '__main__':
obj = MyClass(10)
print(obj.result_matrix)
How can i avoid launching the processes that correspond to the lower half (or the upper half) of the matrix ?