Activation parameter not working in GridSearch

Question

I am trying to make a GridSearch for best parameters, like this:

def MultiPerceptron(optimizer = 'adam', loss = 'binary_cross_entropy', kernel_initializer = 'random_uniform', activation = 'relu', units = 16):
  model = Sequential()
  model.add(InputLayer(30))
  model.add(Dense(units = units, activation = activation, kernel_initializer = kernel_initializer))
  model.add(Dense(units = units, activation = activation, kernel_initializer = kernel_initializer))
  model.add(Dense(units = 1, activation = 'sigmoid'))
  model.compile(optimizer = optimizer, loss = loss, metrics =['binary_accuracy'])
  return model

classifier = KerasClassifier(build_fn = MultiPerceptron, validation_split = 0.1, validation_batch_size = 50)
param = {'batch_size': [10, 30],
         'epochs': [50, 100],
         'optimizer': ['adam', 'sgd'],
         'loss': ['binary_crossentropy', 'hinge'],
         'kernel_initializer': ['random_uniform', 'normal'],
         'activation': ['relu', 'tanh'],
         'units': [16, 8]}

search = GridSearchCV(estimator = classifier, param_grid = param, scoring = 'accuracy', cv = 5)
search = search.fit(x,y)

And i am getting the following error:

ValueError: Invalid parameter activation for estimator KerasClassifier.
This issue can likely be resolved by setting this parameter in the KerasClassifier constructor:
`KerasClassifier(activation=relu)`
Check the list of available parameters with `estimator.get_params().keys()`

@trinity420 Yes, passing the parameter `activation=relu` to `KerasClassifier` solved my problem. — Murilo, Apr 13 '22 at 21:50

score 1 · Answer 1 · answered Oct 31 '22 at 11:09

Use activation and layers in KerasClassifier constructor

def create_model(layers, activation):
        model= Sequential()
        for i, nodes in enumerate(layers):
            if i==0:
                model.add(Dense(nodes, input_dim=X_train.shape[1]))
                model.add(Activation(activation))
            else:
                model.add(Dense(nodes))
                model.add(Activation(activation))
        model.add(Dense(1))
        
        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
        return model
    
    model= KerasClassifier(model=create_model, verbose=0, activation='relu', layers=20)

Then

layers=[[20],[40,20], [45, 30, 15]]
activations = ['sigmoid','relu']
param_grid = dict(layers=layers, activation=activations, batch_size=[128, 256], epochs=[30])
grid = GridSearchCV(estimator=model, param_grid=param_grid)

grid_result= grid.fit(X_train, y_train)
[grid_result.best_score_,grid_result.best_params_]

Its worked! Finally got below output:

[0.8397500000000001,
 {'activation': 'relu',
  'batch_size': 128,
  'epochs': 30,
  'layers': [45, 30, 15]}]

score 0 · Answer 2 · answered Dec 07 '21 at 10:40

0

I think they changed something, because i could only make it work passing the activation=relu parameter to KerasClassifier.

Other parameters aren't needed there.

answered Dec 07 '21 at 10:40

Murilo

533
3
15

score 0 · Answer 3 · answered Jan 22 '22 at 08:05

I had the same issue. The following code ran perfectly when using keras.wrappers

def build_model(lambda_parameter):
    model = Sequential()
    model.add(Dense(10, input_dim=X.shape[1], activation='relu', 
    kernel_regularizer=l2(lambda_parameter)))
    model.add(Dense(6, activation='relu', 
    kernel_regularizer=l2(lambda_parameter)))
    model.add(Dense(4, activation='relu', 
    kernel_regularizer=l2(lambda_parameter)))
    model.add(Dense(1, activation='sigmoid'))
    model.compile(loss='binary_crossentropy', optimizer='sgd', metrics= 
    ['accuracy'])
    return model
seed = 1
np.random.seed(seed)
random.set_seed(seed)
model = KerasClassifier(build_fn=build_model, verbose=0, shuffle=False)
lambda_parameter = [0.01, 0.5, 1]
epochs = [50, 100]
batch_size = [20]
param_grid = dict(lambda_parameter=lambda_parameter, epochs=epochs, 
batch_size=batch_size)
grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=5)
results_1 = grid_search.fit(X, y)
print(f"Best cross-validation score = {results_1.best_score_}")
print(f"Parameters for best cross-validation score = 
{results_1.best_params_}")
accuracy_means = results_1.cv_results_['mean_test_score']
accuracy_stds = results_1.cv_results_['std_test_score']
parameters = results_1.cv_results_['params']
for p in range(len(parameters)):
    print(f"Accuracy {accuracy_means[p]} for params {accuracy_stds[p]}, 
    {parameters[p]}

But after switching to Scikeras, I was always getting the a ValueError:

ValueError: Invalid parameter lambda_parameter for estimator 
KerasClassifier.
This issue can likely be resolved by setting this parameter in the 
KerasClassifier constructor: KerasClassifier(lambda_parameter=0.01)`
Check the list of available parameters with 
estimator.get_params().keys()`

I added lambda_parameter=0.01 to the KerasClassifiet to solve the issue

model = KerasClassifier(model=build_model, verbose=0, shuffle=False, 
lambda_parameter=0.01)

Sauron · Answer 4 · 2023-08-15T17:44:18.910

I had the same error yesterday, I found this on scikeras github repo
If you are using scikeras.wrappers.KerasClassifier, you will have to define your parameters like model__activation=['relu','softmax'] , model with 2 underscores then parameter name.
see scikeras official documentation.
https://adriangb.com/scikeras/stable/quickstart.html#grid-search https://adriangb.com/scikeras/stable/migration.html#default-arguments-in-build-fn-model

params={
'batch_size':[20,25],
'epochs':[50,70],
'model__neurons_1':[6,7],
'model__neurons_2':[4,3],
'model__activation':['relu','softmax'],
'model__optimizer':['adam','rmsprop'],
'model__dropout':[0.1,0.2]
}

def create_model(neurons_1,neurons_2,activation,optimizer,dropout):
    nn = tf.keras.Sequential()
    nn.add(tf.keras.layers.Input(shape=11))       
    nn.add(tf.keras.layers.Dense(units=neurons_1,
           activation=activation,kernel_initializer='glorot_uniform'))
    nn.add(tf.keras.layers.Dropout(rate=dropout))
    nn.add(tf.keras.layers.Dense(units=neurons_2,activation=activation))    
    nn.add(tf.keras.layers.Dropout(rate=dropout))
    nn.add(tf.keras.layers.Dense(units=1,activation='sigmoid'))
    nn.compile(optimizer=optimizer,loss='binary_crossentropy',
               metrics=['accuracy']) 
    return nn

model = KerasClassifier(model=create_model)

gs= GridSearchCV(estimator=model,param_grid=params,scoring='accuracy',cv=10,
                 n_jobs=-1,return_train_score=True,verbose=0)

Activation parameter not working in GridSearch

4 Answers4

Linked