Only the clever can figure out!
If somebody works out the scale_pos_weight in XGBoost, after finding out the best point, if they refit the model with that point, another figure comes up! i.e.: Best: 0.950298 using {'scale_pos_weight': 1.028} Baseline (raw) ROC_AUC = 0.87857
Whereas we expected Baseline (raw) ROC_AUC = 0.950298
Why?!
A very simpled stand-alone script is as:
`
`import numpy as np
from sklearn import datasets
from sklearn.model_selection import cross_val_score
from xgboost import XGBClassifier
from collections import Counter
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RepeatedStratifiedKFold
X, y = datasets.load_iris(return_X_y=True)
y00=y
ee=-1
for jj in y:
ee+=1
if y[ee]==2:
y[ee]=np.random.randint(0,2)
counter = Counter(y)
# estimate scale_pos_weight value
Nsbat = counter[0] / counter[1]
#%%
# from numpy import mean
# from sklearn.datasets import make_classification
# define model
modelXGB = XGBClassifier(use_label_encoder=False,\
eval_metric='auc')
# define grid
w0 =np.linspace(0.2,1.1,201)
w1=np.append(w0,Nsbat)
w2=np.append(w1,1)
weights =np.sort(w2)
param_grid = dict(scale_pos_weight=weights)
# define evaluation procedure
cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
# define grid search
grid = GridSearchCV(estimator=modelXGB, param_grid=param_grid, n_jobs=-1, cv=cv, scoring='roc_auc')
# execute the grid search
grid_result = grid.fit(X, y)
# report the best configuration
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
#%%
res_imb=np.array(list(grid_result.best_params_.values()))[0]
modelXGB.scale_pos_weight=res_imb
cvs=cross_val_score(modelXGB, X, y, scoring='roc_auc')
baseline = cvs.mean()
print(f'Baseline (raw) ROC_AUC = {baseline:.5f}')
`
`
