Task5——模型融合

數據競賽過程中若僅採用一種方法是不能得到很好的效果的，通常要進行模型融合。模型融合通常包括3種方式：

• 簡單加權融合：（1）迴歸：算數平均融合、幾何平均融合；（2）分類：投票；（3）綜合：排序融合、log融合。
• stacking/blending：構建多層模型，並利用預測結果再擬合預測。
• boosting/bagging：多樹的提升方法。

stacking理論

stacking是將用初始訓練數據學習出若干個基學習器，將這些學習器的預測結果作爲新的訓練集來學習的一個新學習器。

不同類型的實現代碼

1.迴歸、分類概率融合：
（1）簡單加權平均，結果直接融合

import numpy as np
import pandas as pd
## 定義結果的加權平均函數
def Weighted_method(test_pre1,test_pre2,test_pre3,w=[1/3,1/3,1/3]):
	Weighted_result = w[0]*pd.Series(test_pre1)+w[1]*pd.Series(test_pre2)+w[2]*pd.Series(test_pre3)
	return Weighted_result

from sklearn import metrics
# 各模型的預測結果計算MAE
print('Pred1 MAE:',metrics.mean_absolute_error(y_test_true, test_pre1))
print('Pred2 MAE:',metrics.mean_absolute_error(y_test_true, test_pre2))
print('Pred3 MAE:',metrics.mean_absolute_error(y_test_true, test_pre3))

## 根據加權計算MAE
w = [0.3,0.4,0.3] # 定義比重權值
Weighted_pre = Weighted_method(test_pre1,test_pre2,test_pre3,w)
print('Weighted_pre MAE:',metrics.mean_absolute_error(y_test_true, Weighted_pre))

#Mean平均
## 定義結果的加權平均函數
def Mean_method(test_pre1,test_pre2,test_pre3):
	Mean_result = pd.concat([pd.Series(test_pre1),pd.Series(test_pre2),pd.Series(test_pre3)],axis=1)
	return Mean_result
Mean_pre = Mean_method(test_pre1,test_pre2,test_pre3)
print('Mean_pre MAE:',metrics.mean_absolute_error(y_test_true, Mean_pre))

#median平均
## 定義結果的加權平均函數
def Median_method(test_pre1,test_pre2,test_pre3):
	Median_result = pd.concat([pd.Series(test_pre1),pd.Series(test_pre2),pd.Series(test_pre3)],axis=
	return Median_result
Median_pre = Median_method(test_pre1,test_pre2,test_pre3)
print('Median_pre MAE:',metrics.mean_absolute_error(y_test_true, Median_pre))

（2）Stacking融合(迴歸)

from sklearn import linear_model
def Stacking_method(train_reg1,train_reg2,train_reg3,y_train_true,test_pre1,test_pre2,test_pre3,mode
	model_L2.fit(pd.concat([pd.Series(train_reg1),pd.Series(train_reg2),pd.Series(train_reg3)],axis=
	Stacking_result = model_L2.predict(pd.concat([pd.Series(test_pre1),pd.Series(test_pre2),pd.Serie
	return Stacking_result
model_L2= linear_model.LinearRegression()
Stacking_pre = Stacking_method(train_reg1,train_reg2,train_reg3,y_train_true,
test_pre1,test_pre2,test_pre3,model_L2)
print('Stacking_pre MAE:',metrics.mean_absolute_error(y_test_true, Stacking_pre))

2.分類模型融合
（1）Voting投票機制（少數服從多數/ 硬投票、軟投票）

'''
硬投票：對多個模型直接進行投票，不區分模型結果的相對重要度，最終投票數最多的類爲最終被預測的類。
'''
iris = datasets.load_iris()
x=iris.data
y=iris.target
x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.3)
clf1 = XGBClassifier(learning_rate=0.1, n_estimators=150, max_depth=3, min_child_weight=2, subsampl
colsample_bytree=0.6, objective='binary:logistic')
clf2 = RandomForestClassifier(n_estimators=50, max_depth=1, min_samples_split=4,
min_samples_leaf=63,oob_score=True)
clf3 = SVC(C=0.1)
# 硬投票
eclf = VotingClassifier(estimators=[('xgb', clf1), ('rf', clf2), ('svc', clf3)], voting='hard')
for clf, label in zip([clf1, clf2, clf3, eclf], ['XGBBoosting', 'Random Forest', 'SVM', 'Ensemble']
	scores = cross_val_score(clf, x, y, cv=5, scoring='accuracy')
	print("Accuracy: %0.2f (+/- %0.2f) [%s]" % (scores.mean(), scores.std(), label))
'''
軟投票：和硬投票原理相同，增加了設置權重的功能，可以爲不同模型設置不同權重，進而區別模型不同的重要度
'''
x=iris.data
y=iris.target
x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.3)
clf1 = XGBClassifier(learning_rate=0.1, n_estimators=150, max_depth=3, min_child_weight=2, subsampl
colsample_bytree=0.8, objective='binary:logistic')
clf2 = RandomForestClassifier(n_estimators=50, max_depth=1, min_samples_split=4,
min_samples_leaf=63,oob_score=True)
clf3 = SVC(C=0.1, probability=True)
# 軟投票
eclf = VotingClassifier(estimators=[('xgb', clf1), ('rf', clf2), ('svc', clf3)], voting='soft', weig
clf1.fit(x_train, y_train)
for clf, label in zip([clf1, clf2, clf3, eclf], ['XGBBoosting', 'Random Forest', 'SVM', 'Ensemble']
	scores = cross_val_score(clf, x, y, cv=5, scoring='accuracy')
	print("Accuracy: %0.2f (+/- %0.2f) [%s]" % (scores.mean(), scores.std(), label))

（2）分類的stacking/blending融合

'''
5-Fold Stacking
'''
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier,GradientBoostingClassifier
import pandas as pd
#創建訓練的數據集
data_0 = iris.data
data = data_0[:100,:]
target_0 = iris.target
target = target_0[:100]
#模型融合中使用到的各個單模型
clfs = [LogisticRegression(solver='lbfgs'),
RandomForestClassifier(n_estimators=5, n_jobs=-1, criterion='gini'),
ExtraTreesClassifier(n_estimators=5, n_jobs=-1, criterion='gini'),
ExtraTreesClassifier(n_estimators=5, n_jobs=-1, criterion='entropy'),
GradientBoostingClassifier(learning_rate=0.05, subsample=0.5, max_depth=6, n_estimators=5)]
#切分一部分數據作爲測試集
X, X_predict, y, y_predict = train_test_split(data, target, test_size=0.3, random_state=2020)
dataset_blend_train = np.zeros((X.shape[0], len(clfs)))
dataset_blend_test = np.zeros((X_predict.shape[0], len(clfs)))
#5折stacking
n_splits = 5
skf = StratifiedKFold(n_splits)
skf = skf.split(X, y)
for j, clf in enumerate(clfs):
#依次訓練各個單模型
	dataset_blend_test_j = np.zeros((X_predict.shape[0], 5))
	for i, (train, test) in enumerate(skf):
#5-Fold交叉訓練，使用第i個部分作爲預測，剩餘的部分來訓練模型，獲得其預測的輸出作爲第i部分的新
		X_train, y_train, X_test, y_test = X[train], y[train], X[test], y[test]
		clf.fit(X_train, y_train)
		y_submission = clf.predict_proba(X_test)[:, 1]
		dataset_blend_train[test, j] = y_submission
		dataset_blend_test_j[:, i] = clf.predict_proba(X_predict)[:, 1]
#對於測試集，直接用這k個模型的預測值均值作爲新的特徵。
	dataset_blend_test[:, j] = dataset_blend_test_j.mean(1)
	print("val auc Score: %f" % roc_auc_score(y_predict, dataset_blend_test[:, j]))
clf = LogisticRegression(solver='lbfgs')
clf.fit(dataset_blend_train, y)
y_submission = clf.predict_proba(dataset_blend_test)[:, 1]
print("Val auc Score of Stacking: %f" % (roc_auc_score(y_predict, y_submission)))

'''
Blending
'''
#創建訓練的數據集
#創建訓練的數據集
data_0 = iris.data
data = data_0[:100,:]
target_0 = iris.target
target = target_0[:100]
#模型融合中使用到的各個單模型
clfs = [LogisticRegression(solver='lbfgs'),
RandomForestClassifier(n_estimators=5, n_jobs=-1, criterion='gini'),
RandomForestClassifier(n_estimators=5, n_jobs=-1, criterion='entropy'),
ExtraTreesClassifier(n_estimators=5, n_jobs=-1, criterion='gini'),
#ExtraTreesClassifier(n_estimators=5, n_jobs=-1, criterion='entropy'),
GradientBoostingClassifier(learning_rate=0.05, subsample=0.5, max_depth=6, n_estimators=5)]
#切分一部分數據作爲測試集
X, X_predict, y, y_predict = train_test_split(data, target, test_size=0.3, random_state=2020)
#切分訓練數據集爲d1,d2兩部分
X_d1, X_d2, y_d1, y_d2 = train_test_split(X, y, test_size=0.5, random_state=2020)
dataset_d1 = np.zeros((X_d2.shape[0], len(clfs)))
dataset_d2 = np.zeros((X_predict.shape[0], len(clfs)))
for j, clf in enumerate(clfs):
#依次訓練各個單模型
	clf.fit(X_d1, y_d1)
	y_submission = clf.predict_proba(X_d2)[:, 1]
	dataset_d1[:, j] = y_submission
#對於測試集，直接用這k個模型的預測值作爲新的特徵。
	dataset_d2[:, j] = clf.predict_proba(X_predict)[:, 1]
	print("val auc Score: %f" % roc_auc_score(y_predict, dataset_d2[:, j]))
#融合使用的模型
clf = GradientBoostingClassifier(learning_rate=0.02, subsample=0.5, max_depth=6, n_estimators=30)
clf.fit(dataset_d1, y_d2)
y_submission = clf.predict_proba(dataset_d2)[:, 1]
print("Val auc Score of Blending: %f" % (roc_auc_score(y_predict, y_submission)))

（3）分類的stacking融合（利用mlxtend）

import warnings
warnings.filterwarnings('ignore')
import itertools
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from sklearn import datasets
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from mlxtend.classifier import StackingClassifier
from sklearn.model_selection import cross_val_score
from mlxtend.plotting import plot_learning_curves
from mlxtend.plotting import plot_decision_regions
# 以python自帶的鳶尾花數據集爲例
iris = datasets.load_iris()
X, y = iris.data[:, 1:3], iris.target
clf1 = KNeighborsClassifier(n_neighbors=1)
clf2 = RandomForestClassifier(random_state=1)
clf3 = GaussianNB()
lr = LogisticRegression()
sclf = StackingClassifier(classifiers=[clf1, clf2, clf3],
meta_classifier=lr)
label = ['KNN', 'Random Forest', 'Naive Bayes', 'Stacking Classifier']
clf_list = [clf1, clf2, clf3, sclf]
fig = plt.figure(figsize=(10,8))
gs = gridspec.GridSpec(2, 2)
grid = itertools.product([0,1],repeat=2)
clf_cv_mean = []
clf_cv_std = []
for clf, label, grd in zip(clf_list, label, grid):
	scores = cross_val_score(clf, X, y, cv=3, scoring='accuracy')
	print("Accuracy: %.2f (+/- %.2f) [%s]" %(scores.mean(), scores.std(), label))
	clf_cv_mean.append(scores.mean())
	clf_cv_std.append(scores.std())
	clf.fit(X, y)
	ax = plt.subplot(gs[grd[0], grd[1]])
	fig = plot_decision_regions(X=X, y=y, clf=clf)
	plt.title(label)
plt.show()