How to use cross_cal_score for MNIST dataset? Shapes between the output of my model and the y in cross_cal_score are incompatible.
I would like to perform k-fold cross validation for MNIST dataset (an example in DEEP LEARNING: FROM BASICS TO PRACTICE).
Here is the softmax layer of the model(10 outputs):
model.add(Dense(10, activation='softmax'))
And I use 'cross_val_score' and keras wrapper 'KerasClassifier'
# make a model and wrap it up for scikit-learn
kc_model = KerasClassifier(model=make_model,
number_of_layers=2, neurons_per_layer=32, optimizer='adam',
epochs=100, batch_size=256, verbose=1)
# create cross-validator
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=random_seed)
results = cross_val_score(estimator=kc_model, X=X_train, y=original_y_train, cv=kfold, verbose=1)
After running the code I get a warning advising the incompatible shapes between the output of my model and the y in cross_cal_score. because the original_y_train is NOT a one-hot version.
ValueError: Shapes (None, 1) and (None, 10) are incompatible
How can I perform k-fold cross-validation for a classifier?
Here is the my code below:
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# convert to floating-point
X_train = keras_backend.cast_to_floatx(X_train)
number_of_pixels = 28 * 28
# normalization: scale data to range[0, 1]
X_train /= 255
# save the original y_train and y_test
original_y_train = y_train
# replace label data with one-hot encoded versions
number_of_classes = 1 + max(np.append(y_train, y_test))
# reshape samples to 2D grid, one line per image
X_train = X_train.reshape([X_train.shape[0], number_of_pixels])
def make_model(number_of_layers=2, neurons_per_layer=32, optimizer='adam', dropout_ratio=0.2):
# create a model
model = Sequential()
# the first layer
model.add(Dense(neurons_per_layer, activation='relu', input_shape=[number_of_pixels], kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_ratio))
# the following layer
for i in range(number_of_layers - 1):
model.add(Dense(32, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_ratio))
# the softmax layer
model.add(Dense(number_of_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
return model
# make a model and wrap it up for scikit-learn
kc_model = KerasClassifier(model=make_model,
number_of_layers=2, neurons_per_layer=32, optimizer='adam',
epochs=100, batch_size=256, verbose=1)
# create cross-validator
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=random_seed)
results = cross_val_score(estimator=kc_model, X=X_train, y=original_y_train, cv=kfold, verbose=1)
And the warning:
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
Epoch 1/100
[Parallel(n_jobs=1)]: Done 10 out of 10 | elapsed: 4.1s finished
Traceback (most recent call last):
File "D:\Python\keras\cross_val_model.py", line 65, in <module>
results = cross_val_score(estimator=kc_model, X=X_train, y=original_y_train, cv=kfold, verbose=1)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\sklearn\model_selection\_validation.py", line 515, in cross_val_score
cv_results = cross_validate(
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\sklearn\model_selection\_validation.py", line 285, in cross_validate
_warn_or_raise_about_fit_failures(results, error_score)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\sklearn\model_selection\_validation.py", line 367, in _warn_or_raise_about_fit_failures
raise ValueError(all_fits_failed_message)
ValueError:
All the 10 fits failed.
It is very likely that your model is misconfigured.
You can try to debug the error by setting error_score='raise'.
Below are more details about the failures:
--------------------------------------------------------------------------------
10 fits failed with the following error:
Traceback (most recent call last):
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\sklearn\model_selection\_validation.py", line 686, in _fit_and_score
estimator.fit(X_train, y_train, **fit_params)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\scikeras\wrappers.py", line 1494, in fit
super().fit(X=X, y=y, sample_weight=sample_weight, **kwargs)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\scikeras\wrappers.py", line 762, in fit
self._fit(
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\scikeras\wrappers.py", line 931, in _fit
self._fit_keras_model(
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\scikeras\wrappers.py", line 526, in _fit_keras_model
hist = self.model_.fit(x=X, y=y, **fit_args)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\18621\AppData\Local\Temp\__autograph_generated_filesuem1zgk.py", line 15, in tf__train_function
retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
ValueError: in user code:
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\engine\training.py", line 1051, in train_function *
return step_function(self, iterator)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\engine\training.py", line 1040, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\engine\training.py", line 1030, in run_step **
outputs = model.train_step(data)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\engine\training.py", line 890, in train_step
loss = self.compute_loss(x, y, y_pred, sample_weight)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\engine\training.py", line 948, in compute_loss
return self.compiled_loss(
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\engine\compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\losses.py", line 139, in __call__
losses = call_fn(y_true, y_pred)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\losses.py", line 243, in call **
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\losses.py", line 1787, in categorical_crossentropy
return backend.categorical_crossentropy(
File "C:\Users\18621\anaconda3\envs\TensorFLow\lib\site-packages\keras\backend.py", line 5119, in categorical_crossentropy
target.shape.assert_is_compatible_with(output.shape)
ValueError: Shapes (None, 1) and (None, 10) are incompatible
