python DEAP library Multiple Processors

Question

I want to solve a multi-objective optimization problem using DEAP, a python based framework. Due to time consuming processes, i need to use all of my CPU power to compute. So i used multiprocessing library as suggested in DEAP documentation an this example, but it results in PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed . My total code is too long to write it down hear, but the following code is similar to my code and results in the same error.Can you please tell me where do i make mistake? Thanks in advance

import multiprocessing
from deap import creator, base, tools, algorithms
import random
import matplotlib.pyplot as plt
def TEST(dec_var):
    return dec_var[0]**2+dec_var[1]**2,(dec_var[0]-2)**2+dec_var[1]**2
def feasible(dec_var):
    if all(i>0 for i in dec_var):
        return True
    return False

creator.create("FitnessMin", base.Fitness, weights=(-1.0,-1.0))
creator.create("Individual", list, fitness=creator.FitnessMin)
toolbox=base.Toolbox()

toolbox.register("uniform", random.uniform, 0.0, 7.0)
toolbox.register("individual",tools.initRepeat,creator.Individual,toolbox.uniform ,n=2)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)

toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutGaussian, mu=0, sigma=1, indpb=0.1)
toolbox.register("select", tools.selNSGA2)
toolbox.register("evaluate", TEST)
toolbox.decorate("evaluate", tools.DeltaPenalty(feasible,(1000,1000)))

def main(seed=None):
    random.seed(seed)

    NGEN = 250
    MU = 100
    CXPB = 0.9

    stats_func1 = tools.Statistics(key=lambda ind: ind.fitness.values[0])
    stats_func2 = tools.Statistics(key=lambda ind: ind.fitness.values[1])
    stats = tools.MultiStatistics(func1=stats_func1, func2=stats_func2)
    stats.register("avg", numpy.mean, axis=0)
    stats.register("std", numpy.std, axis=0)
    stats.register("min", numpy.min, axis=0)
    stats.register("max", numpy.max, axis=0)

    logbook = tools.Logbook()
    logbook.header = "gen", "evals", "func1","func2"
    logbook.chapters["func1"].header = "min", "max"
    logbook.chapters["func2"].header = "min", "max"    
    pop = toolbox.population(n=MU)


    invalid_ind = [ind for ind in pop if not ind.fitness.valid]
    fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
    for ind, fit in zip(invalid_ind, fitnesses):
        ind.fitness.values = fit


    pop = toolbox.select(pop, len(pop))
    record = stats.compile(pop)
    logbook.record(gen=0, evals=len(invalid_ind), **record)
    print(logbook.stream)


    for gen in range(1, NGEN):

        offspring = tools.selTournamentDCD(pop, len(pop))
        offspring = [toolbox.clone(ind) for ind in offspring]

        for ind1, ind2 in zip(offspring[::2], offspring[1::2]):
            if random.random() <= CXPB:
                toolbox.mate(ind1, ind2)

            toolbox.mutate(ind1)
            toolbox.mutate(ind2)
            del ind1.fitness.values, ind2.fitness.values


        invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
        fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
        for ind, fit in zip(invalid_ind, fitnesses):
            ind.fitness.values = fit


        pop = toolbox.select(pop + offspring, MU)
        record = stats.compile(pop)
        logbook.record(gen=gen, evals=len(invalid_ind), **record)
        print(logbook.stream)



    return pop, logbook
if __name__ == "__main__":
    pool = multiprocessing.Pool(processes=4)
    toolbox.register("map", pool.map)
    pop, stats = main()
    pool.close()
    print pop

score 0 · Answer 1 · answered Feb 17 '20 at 14:20

0

This is noted on the documentation page you linked:

The pickling of lambda function is not yet available in Python.

You are using lambda functions in stats_func1 and stats_func2 – just move them to global functions and try again:

def stats_key_1(ind):
    return ind.fitness.values[0]

def stats_key_2(ind):
    return ind.fitness.values[1]

# ... snip ...

def main(seed=None):

    # ... snip ...

    stats_func1 = tools.Statistics(key=stats_key_1)
    stats_func2 = tools.Statistics(key=stats_key_1)

answered Feb 17 '20 at 14:20

AKX

152,115
15
115
172

I appreciate your help. It seems that moving functions to global functions prevents from raising error, however it doesn't yield multi-core processing. Still just one core works. – Sina_Alef Feb 22 '20 at 07:56
Hi @Sina_Alef I have the same problem. Did you solved it? – CodeIK Oct 01 '20 at 20:10

python DEAP library Multiple Processors

1 Answers1