## In many applications, we don't abuse test set like that
## we use cross validation to replace multiple evaluations on test set

## To do CV, there is no need to write multiple loops all by myself
from sklearn.model_selection import GridSearchCV

parameters_to_search = {'learning_rate': learning_rates, 
              'min_samples_leaf': min_samples_leafs}

gb_model = GradientBoostingRegressor(n_estimators = 300, 
                                     subsample = 0.7,
                                    n_iter_no_change = 10,
                                     random_state = randomState)

gb_model_CV = GridSearchCV(gb_model, parameters_to_search, cv=5)
gb_model_CV.fit(X_train.fillna(-1), y_train)


## the gridcv module run the models and save the results for us
gb_model_CV.cv_results_

## the mean of 5-folds test(not true test) R2
gb_model_CV.cv_results_["mean_test_score"]

## the best one is learning_rate=0.18999999999999997, min_samples_leaf=5
gb_model_CV.best_estimator_

# 1. 引入相關套件 pip install optuna 
import optuna
from xgboost import XGBClassifier
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split

def objective(trial):
    # 2. 想要控制哪些參數 "參數名稱", "參數範圍"
    params = {
        'eta': trial.suggest_float("eta", 1e-8, 1.0, log=True),
        'alpha': trial.suggest_float('alpha', 1e-8, 1.0, log=True),
        'lambda': trial.suggest_float('lambda', 1e-8, 1.0, log=True),
        'grow_policy': trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"]),
        'n_estimators': trial.suggest_int('n_estimators', 50, 500),
        'max_depth': trial.suggest_int('max_depth', 3, 10),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),
        'subsample': trial.suggest_float('subsample', 0.5, 1.0),
        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),
        'gamma': trial.suggest_float('gamma', 0, 5),
        'min_child_weight': trial.suggest_int('min_child_weight', 1, 10)
    }
    
    # 3. 初始化並訓練XGBoost模型
    xgb_model = XGBClassifier(
        **params,
        use_label_encoder=False, 
        eval_metric='mlogloss'
    )
    xgb_model.fit(X_train, y_train)
    
    # 4. 預測並計算精確度作為目標
    y_pred_xgboost = xgb_model.predict(X_test)
    accuracy = accuracy_score(y_test, y_pred_xgboost)
    
    return accuracy

# 5. 使用optuna進行超參數調整
study = optuna.create_study(direction='maximize')  # 我們想要最大化精確度
study.optimize(objective, n_trials=50)  # 進行50次試驗

# 6. 獲取最佳超參數
best_params = study.best_params
print("Best hyperparameters: ", best_params)

# 7. 使用最佳超參數訓練最終模型
xgb_model = XGBClassifier(
    **best_params,
    use_label_encoder=False, 
    eval_metric='mlogloss'
)
xgb_model.fit(X_train, y_train)

# 8. 測試最終模型
y_pred_xgboost = xgb_model.predict(X_test)

# 9. 評估最終模型
print("Accuracy (XGBoost):", accuracy_score(y_test, y_pred_xgboost))
print(classification_report(y_test, y_pred_xgboost, target_names=label_encoder.classes_))

from optuna.visualization import plot_param_importances, plot_optimization_history

# 10. Optuna 視覺化：參數重要性
fig = plot_param_importances(study)
plotly_config = {"staticPlot": True}
fig.show(config=plotly_config)

# 11. Optuna 視覺化：優化歷史
fig = plot_optimization_history(study)
fig.show(config=plotly_config)