Xgboost生成Feature

瞭解了一個新的方法

坐下記錄，不過發現這樣新特徵加入之後，訓練的結果並沒有提升非常多。

from sklearn.model_selection import train_test_split
from pandas import DataFrame
from sklearn import metrics
from sklearn.datasets  import  make_hastie_10_2
from xgboost.sklearn import XGBClassifier
import xgboost as xgb

#準備數據，y本來是[-1:1],xgboost自帶接口邀請標籤是[0:1],把-1的轉成1了。
X, y = make_hastie_10_2(random_state=0)
X = DataFrame(X)
y = DataFrame(y)
y.columns={"label"}
label={-1:0,1:1}
y.label=y.label.map(label)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) #劃分數據集
y_train.head(5)


print("X_train =")
print(X_train.head(5))

print("X_test =")
X_test.head(5)

#XGBoost自帶接口
params={
    'eta': 0.3,
    'max_depth':3,   
    'min_child_weight':1,
    'gamma':0.3, 
    'subsample':0.8,
    'colsample_bytree':0.8,
    'booster':'gbtree',
    'objective': 'binary:logistic',
    'nthread':12,
    'scale_pos_weight': 1,
    'lambda':1,  
    'seed':27,
    'silent':0 ,
    'eval_metric': 'auc'
}
d_train = xgb.DMatrix(X_train, label=y_train)
d_valid = xgb.DMatrix(X_test, label=y_test)
d_test = xgb.DMatrix(X_test)
watchlist = [(d_train, 'train'), (d_valid, 'valid')]
 
#sklearn接口
clf = XGBClassifier(
    n_estimators=30,#三十棵樹
    learning_rate =0.3,
    max_depth=3,
    min_child_weight=1,
    gamma=0.3,
    subsample=0.8,
    colsample_bytree=0.8,
    objective= 'binary:logistic',
    nthread=12,
    scale_pos_weight=1,
    reg_lambda=1,
    seed=27)
 
model_bst = xgb.train(params, d_train, 30, watchlist, early_stopping_rounds=500, verbose_eval=10)
model_sklearn=clf.fit(X_train, y_train)
 
y_bst= model_bst.predict(d_test)
y_sklearn= clf.predict_proba(X_test)[:,1]

print("XGBoost_自帶接口    AUC Score : %f" % metrics.roc_auc_score(y_test, y_bst))
print("XGBoost_sklearn接口 AUC Score : %f" % metrics.roc_auc_score(y_test, y_sklearn))

"""### 生成兩組新特徵"""

print("原始train大小：",X_train.shape)
print("原始test大小：",X_test.shape)
 
##XGBoost自帶接口生成的新特徵
train_new_feature= model_bst.predict(d_train, pred_leaf=True)
test_new_feature= model_bst.predict(d_test, pred_leaf=True)

train_new_feature1 = DataFrame(train_new_feature)
test_new_feature1 = DataFrame(test_new_feature)
print("新的特徵集(自帶接口)：",train_new_feature1.shape)
print("新的測試集(自帶接口)：",test_new_feature1.shape)
 
#sklearn接口生成的新特徵
train_new_feature= clf.apply(X_train) # 每個樣本在每顆樹葉子節點的索引值
test_new_feature= clf.apply(X_test)
train_new_feature2 = DataFrame(train_new_feature)
test_new_feature2 = DataFrame(test_new_feature)


print("test_new_feature1 =")
print(test_new_feature1.head(5))

print("test_new_feature2 =")
test_new_feature2.head(5)

import numpy as np
import pandas as pd

def mergeToOne(df,df_two):
  df = df.reset_index(drop=True)
  df_two = df_two.reset_index(drop=True)

  n = df.shape[1]
  n+=df_two.shape[1]
  print("n = ",n)
  df_new = pd.concat([df,df_two],axis=1)
  df_new.columns = [i for i in range(n)]
  return df_new
 # pd.merge(df,df_two,)
  #return DataFrame(df_new)

#用兩組新的特徵分別訓練，預測
#用XGBoost自帶接口生成的新特徵訓練
new_feature1=clf.fit(train_new_feature1, y_train)
y_new_feature1= clf.predict_proba(test_new_feature1)[:,1]

print("模型訓練後樹模型：生成的新特徵預測結果 AUC Score : %f" % metrics.roc_auc_score(y_test, y_new_feature1))

#用XGBoost自帶接口生成的新特徵訓練
new_feature2=clf.fit(train_new_feature2, y_train)
y_new_feature2= clf.predict_proba(test_new_feature2)[:,1]

print("生成的新特徵預測結果 AUC Score : %f" % metrics.roc_auc_score(y_test, y_new_feature2))


new_train = mergeToOne(X_train,train_new_feature1)
new_test = mergeToOne(X_test,test_new_feature1)

  
print(new_train.head(2))
clf.fit(new_train, y_train)
y_new_pre = clf.predict_proba(new_test)[:,1]

print("40維度 ： 生成的新特徵預測結果 AUC Score : %f" % metrics.roc_auc_score(y_test, y_new_pre))

參考博客

XGBoost Plotting API以及GBDT組合特徵實踐

MachineLearningTrick