Using custom scoring with Multiclass outputs from Keras model returns the same error for cross_val_score or GridSearchCV as below (it's on Iris, so you can run it directly to test):
import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import to_categorical
from keras.wrappers.scikit_learn import KerasClassifier
iris = datasets.load_iris()
X= iris.data
Y = to_categorical(iris.target)
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8, random_state=1000)
def create_model(optimizer='rmsprop'):
model = Sequential()
model.add(Dense(8,activation='relu',input_shape = (4,)))
model.add(Dense(3,activation='softmax'))
model.compile(optimizer = optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
model = KerasClassifier(build_fn=create_model,
epochs=10,
batch_size=5,
verbose=0)
#results = cross_val_score(model, X_train, Y_train, scoring='precision_macro')
param_grid = {'optimizer':('rmsprop','adam')}
grid = GridSearchCV(model,
param_grid=param_grid,
return_train_score=True,
scoring=['accuracy','precision_macro','recall_macro'],
refit='precision_macro')
grid_results = grid.fit(X_train,Y_train)
So I get this error
I bypassed the full stack as you can reproduce it by copying the code above.
ValueError: Classification metrics can't handle a mix of multilabel-indicator and binary targets
When I, remove the scoring parameters, it works.
Is there any way to avoid that and enable having a f1, precision or any custom score ? without having to rewrite my own grid search code, of course.
Thanks for your help
UPDATE : I've just found how to resolve it
First this doc (http://scikit-learn.org/stable/modules/multiclass.html#multilabel-classification-format) show that the one-hot representation used in Keras is interpreted as multilabel in scikit-learn.
Then looking at scikit_learn.py implementing KerasClassifier class : https://github.com/keras-team/keras/blob/master/keras/wrappers/scikit_learn.py
The fit function in the BaseWrapper class includes this line of code :
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
The Wrapper does the categorical transformation by itself.
It seems that Keras, to avoid this issue, due to the difference in the multiclass representation with scikit-learn, can takes a scikit-learn style multiclass [0,1,2,1,0,2] and transform it into categorical representation just for the NN model fit.
So, I simply tried removing the categorical transformation when passing the model to the sklearn functions.
And it work now
import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.utils import to_categorical
from keras.wrappers.scikit_learn import KerasClassifier
iris = datasets.load_iris()
X= iris.data
#Y = to_categorical(iris.target,3)
Y = iris.target
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8, random_state=1000)
def create_model(optimizer='rmsprop'):
model = Sequential()
model.add(Dense(8,activation='relu',input_shape = (4,)))
model.add(Dense(3,activation='softmax'))
model.compile(optimizer = optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
model = KerasClassifier(build_fn=create_model,
epochs=10,
batch_size=5,
verbose=0)
#results = cross_val_score(model, X_train, Y_train, scoring='precision_macro')
param_grid = {'optimizer':('rmsprop','adam')}
grid = GridSearchCV(model,
param_grid=param_grid,
return_train_score=True,
scoring=['precision_macro','recall_macro','f1_macro'],
refit='precision_macro')
grid_results = grid.fit(X_train,Y_train)
