After hyperparameter optimization yields best parameters when i try and reproduce the result there is a significant difference in results based on the same dataset and seed value:
My optimization code is
def objective(trial):
param = {
'tree_method': 'gpu_hist',
'objective': 'binary:logistic',
'eval_metric': 'logloss',
'seed': 1701,
'booster': trial.suggest_categorical("booster", ["gbtree", "dart"]),
#'lambda': trial.suggest_float("lambda", 1e-8, 1.0),
#'alpha': trial.suggest_float("alpha", 1e-8, 1.0),
'learning_rate': trial.suggest_float('learning_rate', 0.01, 1.0),
'n_estimators': trial.suggest_int('n_estimators', 50, 1000, step =10),
'subsample': trial.suggest_float('subsample', 0.4, 1.0, step =0.05),
'colsample_bytree': trial.suggest_float('colsample_bytree', 0.01, 1.0, step = 0.01),
#'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 1.0),
#'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 1.0)
}
if param["booster"] == "gbtree" or param["booster"] == "dart":
param["max_depth"] = trial.suggest_int('max_depth', 1, 20)
param["min_child_weight"] = trial.suggest_int('min_child_weight', 0.01, 10)
#param["gamma"] = trial.suggest_float('gamma', 1e-8, 1.0)
#param["grow_policy"] = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"])
if param["booster"] == "dart":
param["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"])
param["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"])
param["rate_drop"] = trial.suggest_float("rate_drop", 1e-8, 1.0)
param["skip_drop"] = trial.suggest_float("skip_drop", 1e-8, 1.0, log=True)
model = xgb.XGBClassifier(early_stopping_rounds=100, **param)
model.fit(X_trainS, y_train, eval_set=[(X_testS, y_test)], verbose=False)
y_pred = model.predict(X_testS)
precision = precision_score(y_test, y_pred)
print(precision)
if precision == 0.0 or precision == np.nan or precision == np.inf:
return 1.0
return 1-precision # Optuna minimizes the objective; 1 - precision will maximize precision
study = optuna.create_study()
study.optimize(objective, n_trials=1000, n_jobs=1)
print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))
yields:
*
C:\Users####\anaconda3\Lib\site-packages\optuna\samplers_tpe\probability_distributions.py:121: RuntimeWarning: underflow encountered in exp return np.log(np.exp(weighted_log_pdf - max_[:, None]).sum(axis=1)) + max_ C:\Users####\anaconda3\Lib\site-packages\optuna\samplers_tpe_truncnorm.py:170: RuntimeWarning: underflow encountered in _ndtri_exp_single (vectorized) return np.frompyfunc(_ndtri_exp_single, 1, 1)(y).astype(float) C:\Users####\anaconda3\Lib\site-packages\optuna\samplers_tpe_truncnorm.py:55: RuntimeWarning: underflow encountered in exp return log_p + np.log1p(-np.exp(log_q - log_p)) [I 2023-08-10 17:09:24,248] Trial 143 finished with value: 0.43262411347517726 and parameters: {'booster': 'dart', 'learning_rate': 0.19204390762883145, 'n_estimators': 850, 'subsample': 1.0, 'colsample_bytree': 0.9, 'max_depth': 6, 'min_child_weight': 9, 'sample_type': 'uniform', 'normalize_type': 'forest', 'rate_drop': 0.878192554389142, 'skip_drop': 1.2238474618801067e-05}. Best is trial 143 with value: 0.43262411347517726.
and my test code is:
params = {'booster': 'dart', 'learning_rate': 0.19204390762883145, 'n_estimators': 850, 'subsample': 1.0, 'colsample_bytree': 0.9, 'max_depth': 6, 'min_child_weight': 9, 'sample_type': 'uniform', 'normalize_type': 'forest', 'rate_drop': 0.878192554389142, 'skip_drop': 1.2238474618801067e-05}
model = XGBClassifier(**params, random_state=1701)
#model = XGBClassifier(tree_method='gpu_hist',booster ='gbtree',objective='binary:logistic', eval_metric='logloss', colsample_bytree = 0.37, n_estimators=520, learning_rate=0.96,max_depth=19,subsample=0.9,min_child_weight=7, random_state=1701)
model.fit(X_trainS, y_train)
print(confusion_matrix(y_test,model.predict(X_testS)))
print(precision_score(y_test,model.predict(X_testS)))
I am a uncertain to why this should be the case and have included the warnings for the optimal trial incase this is a source of the issue. Any help would be grateful...and note I have treid using early stopping rounds also in my test code but it has not solved the issue
