keras集成學習二-迴歸問題
數據描述
波士頓房價數據集（Boston House Price Dataset）（https://www.kaggle.com/c/boston-housing）
boston_housing數據集對房價數據進行迴歸分析，數據來自1970年代，波斯頓周邊地區的房價，是用於機器學習的經典數據集。該數據集很小，共計506條數據
每條數據包含房屋以及房屋周圍的詳細信息。其中包含城鎮犯罪率，一氧化氮濃度，住宅平均房間數，到中心區域的加權距離以及自住房平均房價等等。
- CRIM：城鎮人均犯罪率。
- ZN：住宅用地超過 25000 sq.ft. 的比例。
- INDUS：城鎮非零售商用土地的比例。
- CHAS：查理斯河空變量（如果邊界是河流，則爲1；否則爲0）。
- NOX：一氧化氮濃度。
- RM：住宅平均房間數。
- AGE：1940 年之前建成的自用房屋比例。
- DIS：到波士頓五個中心區域的加權距離。
- RAD：輻射性公路的接近指數。
- TAX：每 10000 美元的全值財產稅率。
- PTRATIO：城鎮師生比例。
- B：1000（Bk-0.63）^ 2，其中 Bk 指代城鎮中黑人的比例。
- LSTAT：人口中地位低下者的比例。
- MEDV：自住房的平均房價，以千美元計。
keras普通實現
import numpy as np
import matplotlib.pyplot as plt
from keras.datasets import boston_housing
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler
def load_data():
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
x = np.vstack((x_train,x_test))
y = np.concatenate((y_train, y_test))
x = StandardScaler().fit_transform(x)
y = StandardScaler().fit_transform(y.reshape(-1, 1))
x_train = x[1:401, :]
x_test = x[401:, :]
y_train = y[1:401, :]
y_test = y[401:, :]
return (x_train, y_train), (x_test, y_test)
def draw_train_history(history):
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
def build_model():
model = Sequential()
model.add(Dense(64, activation='relu', input_shape=(13,)))
model.add(Dense(64, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(optimizer='adam',
loss='mean_squared_error')
return model
if __name__ == '__main__':
(x_train, y_train), (x_test, y_test) = load_data()
model = build_model()
early_stopping = EarlyStopping(monitor='loss', patience=10)
history = model.fit(x_train, y_train,
epochs=500,
batch_size=64,
validation_split=0.2,
callbacks=[early_stopping])
draw_train_history(history)
loss = model.evaluate(x_test, y_test, batch_size=64)
print("test loss: {}".format(loss))
模型結構
模型輸出
test loss: 0.20772670450664701
模型損失曲線
keras集成學習-平均法
import numpy as np
import matplotlib.pyplot as plt
from keras.datasets import boston_housing
from keras.models import Model
from keras.layers import Input, Dense, average
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler
def load_data():
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
x = np.vstack((x_train,x_test))
y = np.concatenate((y_train, y_test))
x = StandardScaler().fit_transform(x)
y = StandardScaler().fit_transform(y.reshape(-1, 1))
x_train = x[1:401, :]
x_test = x[401:, :]
y_train = y[1:401, :]
y_test = y[401:, :]
return (x_train, y_train), (x_test, y_test)
def draw_train_history(history):
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
def build_model():
inputs = Input(shape=(13, ))
model1_1 = Dense(64, activation='relu')(inputs)
model2_1 = Dense(128, activation='relu')(inputs)
model3_1 = Dense(32, activation='relu')(inputs)
model1_2 = Dense(32, activation='relu')(model1_1)
model2_2 = Dense(64, activation='relu')(model2_1)
model3_2 = Dense(16, activation='relu')(model3_1)
model1_out = Dense(1, activation='linear')(model1_2)
model2_out = Dense(1, activation='linear')(model2_2)
model3_out = Dense(1, activation='linear')(model3_2)
out = average([model1_out, model2_out, model3_out])
model = Model(inputs=inputs, outputs=out)
model.compile(optimizer='adam',
loss='mean_squared_error')
return model
if __name__ == '__main__':
(x_train, y_train), (x_test, y_test) = load_data()
model = build_model()
early_stopping = EarlyStopping(monitor='loss', patience=10)
history = model.fit(x_train, y_train,
epochs=500,
batch_size=64,
validation_split=0.2,
callbacks=[early_stopping])
draw_train_history(history)
model.save("regression-average-ensemble.h5")
loss = model.evaluate(x_test, y_test, batch_size=64)
print("test loss: {}".format(loss))
模型結構
模型輸出
test loss: 0.14132633038929532
模型損失曲線
keras-scikit_learn集成學習
import numpy as np
from keras.datasets import boston_housing
from keras.wrappers.scikit_learn import KerasRegressor
from keras.models import Sequential
from keras.layers import Dense
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import VotingRegressor
from sklearn.externals import joblib
def load_data():
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
x = np.vstack((x_train,x_test))
y = np.concatenate((y_train, y_test))
x = StandardScaler().fit_transform(x)
y = StandardScaler().fit_transform(y.reshape(-1, 1))
x_train = x[1:401, :]
x_test = x[401:, :]
y_train = y[1:401, :]
y_test = y[401:, :]
return (x_train, y_train), (x_test, y_test)
def build_model1():
model = Sequential()
model.add(Dense(128, activation='relu', input_shape=(13, )))
model.add(Dense(64, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(optimizer='adam',
loss='mean_squared_error')
return model
def build_model2():
model = Sequential()
model.add(Dense(64, activation='relu', input_shape=(13, )))
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(optimizer='adam',
loss='mean_squared_error')
return model
def build_model3():
model = Sequential()
model.add(Dense(32, activation='relu', input_shape=(13, )))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation='linear'))
model.compile(optimizer='adam',
loss='mean_squared_error')
return model
if __name__ == '__main__':
(x_train, y_train), (x_test, y_test) = load_data()
model1 = KerasRegressor(build_fn=build_model1, epochs=100, batch_size=64)
model1._estimator_type = "regressor"
model2 = KerasRegressor(build_fn=build_model2, epochs=100, batch_size=64)
model2._estimator_type = "regressor"
model3 = KerasRegressor(build_fn=build_model3, epochs=100, batch_size=64)
model3._estimator_type = "regressor"
cls = VotingRegressor(estimators=[
('model1', model1),
('model2', model2),
('model3', model3)
])
cls.fit(x_train, y_train)
joblib.dump(cls, "sklearn-regressor.h5")
print("score: ", cls.score(x_test, y_test))
這裏我們使用scikit_learn中的VotingRegressor作爲模型集成的工具，在sklearn的文檔中是這樣描述VotingRegressor的
A voting regressor is an ensemble meta-estimator that fits several base regressors, each on the whole dataset. Then it averages the individual predictions to form a final prediction.
也就是它會將每個基學習器的輸出結果做平均作爲最後的輸出結果。
如果我們使用StackingRegressor做模型集成工具的話，他會將每個基學習器的輸出再放進一個regressor中做最後的預測，也就是下面我們要實現的學習法。我們來看一下文檔中的描述
Stacked generalization consists in stacking the output of individual estimator and use a regressor to compute the final prediction. Stacking allows to use the strength of each individual estimator by using their output as input of a final estimator.
模型輸出
score: 0.8482947319781606
keras集成學習-學習法
import numpy as np
import matplotlib.pyplot as plt
from keras.datasets import boston_housing
from keras.models import Model
from keras.layers import Input, Dense, concatenate
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler
def load_data():
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
x = np.vstack((x_train,x_test))
y = np.concatenate((y_train, y_test))
x = StandardScaler().fit_transform(x)
y = StandardScaler().fit_transform(y.reshape(-1, 1))
x_train = x[1:401, :]
x_test = x[401:, :]
y_train = y[1:401, :]
y_test = y[401:, :]
return (x_train, y_train), (x_test, y_test)
def draw_train_history(history):
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
def build_model():
inputs = Input(shape=(13, ))
model1_1 = Dense(64, activation='relu')(inputs)
model2_1 = Dense(128, activation='relu')(inputs)
model3_1 = Dense(32, activation='relu')(inputs)
model1_2 = Dense(32, activation='relu')(model1_1)
model2_2 = Dense(64, activation='relu')(model2_1)
model3_2 = Dense(16, activation='relu')(model3_1)
model1_3 = Dense(1, activation='linear')(model1_2)
model2_3 = Dense(1, activation='linear')(model2_2)
model3_3 = Dense(1, activation='linear')(model3_2)
con = concatenate([model1_3, model2_3, model3_3])
output = Dense(1, activation='linear')(con)
model = Model(inputs=inputs, outputs=output)
model.compile(optimizer='adam',
loss='mean_squared_error')
return model
if __name__ == '__main__':
(x_train, y_train), (x_test, y_test) = load_data()
model = build_model()
early_stopping = EarlyStopping(monitor='loss', patience=10)
history = model.fit(x_train, y_train,
epochs=500,
batch_size=64,
validation_split=0.2,
callbacks=[early_stopping])
draw_train_history(history)
model.save("regression-ensemble.h5")
loss = model.evaluate(x_test, y_test, batch_size=64)
print("test loss: {}".format(loss))
模型結構
模型輸出
test loss: 0.16616634471075875
模型損失曲線
代碼位置
https://github.com/Knowledge-Precipitation-Tribe/Neural-network/tree/master/code/Ensemble-Learning