import pandas as pd
import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import cross_validation, metrics
from sklearn.grid_search import GridSearchCV
import matplotlib.pylab as plt
%matplotlib inline
train = pd.read_csv('train_modified.csv')
target='Disbursed' #Disbursed的值就是二元分类的输出
IDcol = 'ID'
train['Disbursed'].value_counts()
0 19680
1 320
Name: Disbursed, dtype: int64
x_columns = [x for x in train.columns if x not in [target, IDcol]]
X = train[x_columns]
y = train['Disbursed']
gbm0 = GradientBoostingClassifier(random_state=10)
gbm0.fit(X,y)
y_pred = gbm0.predict(X)
y_predprob = gbm0.predict_proba(X)[:,1] #预测结果包含等于0、1的概率
print("Accuracy : %.4g" % metrics.accuracy_score(y.values, y_pred))
print("AUC Score (Train): %f" % metrics.roc_auc_score(y.values, y_predprob))
Accuracy : 0.9852
AUC Score (Train): 0.900531
. . .
param_test1 = {'n_estimators':np.arange(20,81,10)}
gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=300,
min_samples_leaf=20,max_depth=8,max_features='sqrt', subsample=0.8,random_state=10),
param_grid = param_test1, scoring='roc_auc',iid=False,cv=5) #iid交叉验证误差估计不使用平均
gsearch1.fit(X,y)
gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_
([mean: 0.81285, std: 0.01967, params: {'n_estimators': 20},
mean: 0.81438, std: 0.01947, params: {'n_estimators': 30},
mean: 0.81451, std: 0.01933, params: {'n_estimators': 40},
mean: 0.81618, std: 0.01848, params: {'n_estimators': 50},
mean: 0.81779, std: 0.01736, params: {'n_estimators': 60},
mean: 0.81533, std: 0.01900, params: {'n_estimators': 70},
mean: 0.81322, std: 0.01860, params: {'n_estimators': 80}],
{'n_estimators': 60},
0.8177893165650406)
param_test2 = {'max_depth':np.arange(3,14,2), 'min_samples_split':np.arange(100,801,200)}
gsearch2 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60, min_samples_leaf=20,
max_features='sqrt', subsample=0.8, random_state=10),
param_grid = param_test2, scoring='roc_auc',iid=False, cv=5)
gsearch2.fit(X,y)
gsearch2.grid_scores_, gsearch2.best_params_, gsearch2.best_score_
([mean: 0.81199, std: 0.02073, params: {'max_depth': 3, 'min_samples_split': 100},
mean: 0.81267, std: 0.01985, params: {'max_depth': 3, 'min_samples_split': 300},
mean: 0.81238, std: 0.01937, params: {'max_depth': 3, 'min_samples_split': 500},
mean: 0.80925, std: 0.02051, params: {'max_depth': 3, 'min_samples_split': 700},
mean: 0.81846, std: 0.01843, params: {'max_depth': 5, 'min_samples_split': 100},
mean: 0.81630, std: 0.01810, params: {'max_depth': 5, 'min_samples_split': 300},
mean: 0.81315, std: 0.01898, params: {'max_depth': 5, 'min_samples_split': 500},
mean: 0.81262, std: 0.02090, params: {'max_depth': 5, 'min_samples_split': 700},
mean: 0.81807, std: 0.02004, params: {'max_depth': 7, 'min_samples_split': 100},
mean: 0.82137, std: 0.01733, params: {'max_depth': 7, 'min_samples_split': 300},
mean: 0.81703, std: 0.01773, params: {'max_depth': 7, 'min_samples_split': 500},
mean: 0.81383, std: 0.02327, params: {'max_depth': 7, 'min_samples_split': 700},
mean: 0.81107, std: 0.02178, params: {'max_depth': 9, 'min_samples_split': 100},
mean: 0.80944, std: 0.02612, params: {'max_depth': 9, 'min_samples_split': 300},
mean: 0.81476, std: 0.01973, params: {'max_depth': 9, 'min_samples_split': 500},
mean: 0.81601, std: 0.02576, params: {'max_depth': 9, 'min_samples_split': 700},
mean: 0.81101, std: 0.02222, params: {'max_depth': 11, 'min_samples_split': 100},
mean: 0.81309, std: 0.02696, params: {'max_depth': 11, 'min_samples_split': 300},
mean: 0.81713, std: 0.02379, params: {'max_depth': 11, 'min_samples_split': 500},
mean: 0.81347, std: 0.02702, params: {'max_depth': 11, 'min_samples_split': 700},
mean: 0.81484, std: 0.01776, params: {'max_depth': 13, 'min_samples_split': 100},
mean: 0.80825, std: 0.02291, params: {'max_depth': 13, 'min_samples_split': 300},
mean: 0.81923, std: 0.01693, params: {'max_depth': 13, 'min_samples_split': 500},
mean: 0.81382, std: 0.02258, params: {'max_depth': 13, 'min_samples_split': 700}],
{'max_depth': 7, 'min_samples_split': 300},
0.8213724275914632)
param_test3 = {'min_samples_split':np.arange(800,1900,200), 'min_samples_leaf':np.arange(60,101,10)}
gsearch3 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7,
max_features='sqrt', subsample=0.8, random_state=10),
param_grid = param_test3, scoring='roc_auc',iid=False, cv=5)
gsearch3.fit(X,y)
gsearch3.grid_scores_, gsearch3.best_params_, gsearch3.best_score_
([mean: 0.81828, std: 0.02251, params: {'min_samples_leaf': 60, 'min_samples_split': 800},
mean: 0.81731, std: 0.02344, params: {'min_samples_leaf': 60, 'min_samples_split': 1000},
mean: 0.82220, std: 0.02250, params: {'min_samples_leaf': 60, 'min_samples_split': 1200},
mean: 0.81447, std: 0.02125, params: {'min_samples_leaf': 60, 'min_samples_split': 1400},
mean: 0.81495, std: 0.01626, params: {'min_samples_leaf': 60, 'min_samples_split': 1600},
mean: 0.81528, std: 0.02140, params: {'min_samples_leaf': 60, 'min_samples_split': 1800},
mean: 0.81590, std: 0.02517, params: {'min_samples_leaf': 70, 'min_samples_split': 800},
mean: 0.81573, std: 0.02207, params: {'min_samples_leaf': 70, 'min_samples_split': 1000},
mean: 0.82021, std: 0.02521, params: {'min_samples_leaf': 70, 'min_samples_split': 1200},
mean: 0.81512, std: 0.01995, params: {'min_samples_leaf': 70, 'min_samples_split': 1400},
mean: 0.81395, std: 0.02081, params: {'min_samples_leaf': 70, 'min_samples_split': 1600},
mean: 0.81587, std: 0.02082, params: {'min_samples_leaf': 70, 'min_samples_split': 1800},
mean: 0.82064, std: 0.02698, params: {'min_samples_leaf': 80, 'min_samples_split': 800},
mean: 0.81490, std: 0.02475, params: {'min_samples_leaf': 80, 'min_samples_split': 1000},
mean: 0.82009, std: 0.02568, params: {'min_samples_leaf': 80, 'min_samples_split': 1200},
mean: 0.81850, std: 0.02226, params: {'min_samples_leaf': 80, 'min_samples_split': 1400},
mean: 0.81855, std: 0.02099, params: {'min_samples_leaf': 80, 'min_samples_split': 1600},
mean: 0.81666, std: 0.02249, params: {'min_samples_leaf': 80, 'min_samples_split': 1800},
mean: 0.81960, std: 0.02437, params: {'min_samples_leaf': 90, 'min_samples_split': 800},
mean: 0.81560, std: 0.02235, params: {'min_samples_leaf': 90, 'min_samples_split': 1000},
mean: 0.81936, std: 0.02542, params: {'min_samples_leaf': 90, 'min_samples_split': 1200},
mean: 0.81362, std: 0.02254, params: {'min_samples_leaf': 90, 'min_samples_split': 1400},
mean: 0.81429, std: 0.02417, params: {'min_samples_leaf': 90, 'min_samples_split': 1600},
mean: 0.81299, std: 0.02262, params: {'min_samples_leaf': 90, 'min_samples_split': 1800},
mean: 0.82000, std: 0.02511, params: {'min_samples_leaf': 100, 'min_samples_split': 800},
mean: 0.82209, std: 0.01816, params: {'min_samples_leaf': 100, 'min_samples_split': 1000},
mean: 0.81821, std: 0.02337, params: {'min_samples_leaf': 100, 'min_samples_split': 1200},
mean: 0.81922, std: 0.02377, params: {'min_samples_leaf': 100, 'min_samples_split': 1400},
mean: 0.81545, std: 0.02221, params: {'min_samples_leaf': 100, 'min_samples_split': 1600},
mean: 0.81704, std: 0.02509, params: {'min_samples_leaf': 100, 'min_samples_split': 1800}],
{'min_samples_leaf': 60, 'min_samples_split': 1200},
0.8222032996697154)
gbm1 = GradientBoostingClassifier(learning_rate=0.1, n_estimators=60,max_depth=7, min_samples_leaf =60,
min_samples_split =1200, max_features='sqrt', subsample=0.8, random_state=10)
gbm1.fit(X,y)
y_pred = gbm1.predict(X)
y_predprob = gbm1.predict_proba(X)[:,1]
print("Accuracy : %.4g" % metrics.accuracy_score(y.values, y_pred))
print("AUC Score (Train): %f" % metrics.roc_auc_score(y.values, y_predprob))
Accuracy : 0.984
AUC Score (Train): 0.908099
. . .
param_test4 = {'max_features':np.arange(7,20,2),'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]}
gsearch4 =GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1,n_estimators=60,max_depth=7,min_samples_leaf=60,
min_samples_split =1200, random_state=10),
param_grid = param_test4, scoring='roc_auc',iid=False, cv=5)
gsearch4.fit(X,y)
gsearch4.grid_scores_, gsearch4.best_params_, gsearch4.best_score_
([mean: 0.81678, std: 0.02385, params: {'max_features': 7, 'subsample': 0.6},
mean: 0.81865, std: 0.02132, params: {'max_features': 7, 'subsample': 0.7},
mean: 0.81520, std: 0.02101, params: {'max_features': 7, 'subsample': 0.75},
mean: 0.82220, std: 0.02250, params: {'max_features': 7, 'subsample': 0.8},
mean: 0.82014, std: 0.02062, params: {'max_features': 7, 'subsample': 0.85},
mean: 0.81472, std: 0.02263, params: {'max_features': 7, 'subsample': 0.9},
mean: 0.81828, std: 0.02392, params: {'max_features': 9, 'subsample': 0.6},
mean: 0.82344, std: 0.02708, params: {'max_features': 9, 'subsample': 0.7},
mean: 0.81673, std: 0.02196, params: {'max_features': 9, 'subsample': 0.75},
mean: 0.82241, std: 0.02421, params: {'max_features': 9, 'subsample': 0.8},
mean: 0.82285, std: 0.02446, params: {'max_features': 9, 'subsample': 0.85},
mean: 0.81738, std: 0.02236, params: {'max_features': 9, 'subsample': 0.9},
mean: 0.81513, std: 0.02556, params: {'max_features': 11, 'subsample': 0.6},
mean: 0.81884, std: 0.02175, params: {'max_features': 11, 'subsample': 0.7},
mean: 0.81771, std: 0.02327, params: {'max_features': 11, 'subsample': 0.75},
mean: 0.82108, std: 0.02302, params: {'max_features': 11, 'subsample': 0.8},
mean: 0.81717, std: 0.01998, params: {'max_features': 11, 'subsample': 0.85},
mean: 0.81309, std: 0.02069, params: {'max_features': 11, 'subsample': 0.9},
mean: 0.80983, std: 0.02489, params: {'max_features': 13, 'subsample': 0.6},
mean: 0.81637, std: 0.02087, params: {'max_features': 13, 'subsample': 0.7},
mean: 0.81362, std: 0.01997, params: {'max_features': 13, 'subsample': 0.75},
mean: 0.82064, std: 0.01900, params: {'max_features': 13, 'subsample': 0.8},
mean: 0.81492, std: 0.02091, params: {'max_features': 13, 'subsample': 0.85},
mean: 0.81529, std: 0.01809, params: {'max_features': 13, 'subsample': 0.9},
mean: 0.81891, std: 0.02196, params: {'max_features': 15, 'subsample': 0.6},
mean: 0.81685, std: 0.02028, params: {'max_features': 15, 'subsample': 0.7},
mean: 0.81919, std: 0.02106, params: {'max_features': 15, 'subsample': 0.75},
mean: 0.82198, std: 0.01514, params: {'max_features': 15, 'subsample': 0.8},
mean: 0.81775, std: 0.02571, params: {'max_features': 15, 'subsample': 0.85},
mean: 0.81500, std: 0.02006, params: {'max_features': 15, 'subsample': 0.9},
mean: 0.81498, std: 0.01713, params: {'max_features': 17, 'subsample': 0.6},
mean: 0.81455, std: 0.02273, params: {'max_features': 17, 'subsample': 0.7},
mean: 0.81888, std: 0.01980, params: {'max_features': 17, 'subsample': 0.75},
mean: 0.81355, std: 0.02053, params: {'max_features': 17, 'subsample': 0.8},
mean: 0.82143, std: 0.01901, params: {'max_features': 17, 'subsample': 0.85},
mean: 0.82023, std: 0.01777, params: {'max_features': 17, 'subsample': 0.9},
mean: 0.81647, std: 0.01612, params: {'max_features': 19, 'subsample': 0.6},
mean: 0.81779, std: 0.01749, params: {'max_features': 19, 'subsample': 0.7},
mean: 0.81913, std: 0.02073, params: {'max_features': 19, 'subsample': 0.75},
mean: 0.81877, std: 0.01863, params: {'max_features': 19, 'subsample': 0.8},
mean: 0.81566, std: 0.01775, params: {'max_features': 19, 'subsample': 0.85},
mean: 0.81473, std: 0.01577, params: {'max_features': 19, 'subsample': 0.9}],
{'max_features': 9, 'subsample': 0.7},
0.8234378969766262)
gbm3 = GradientBoostingClassifier(learning_rate=0.01, n_estimators=600,max_depth=7, min_samples_leaf =60,
min_samples_split =1200, max_features=9, subsample=0.7, random_state=10)
gbm3.fit(X,y)
y_pred = gbm3.predict(X)
y_predprob = gbm3.predict_proba(X)[:,1]
print("Accuracy : %.4g" % metrics.accuracy_score(y.values, y_pred))
print("AUC Score (Train): %f" % metrics.roc_auc_score(y, y_predprob))
Accuracy : 0.984
AUC Score (Train): 0.908581
参考链接：http://www.cnblogs.com/DjangoBlog/p/6201663.html#top