TensorFlow2.0從入門到進階——第二章問題總結:wide_deep模型、超參數搜索

看的視頻——https://www.bilibili.com/video/av79196096?p=23

wide_deep模型

1、函數式API實現

單輸入單輸出

input即爲wide模型的輸入、又爲deep模型的輸入 輸出僅有一個output

# model = keras.models.Sequential([
#     keras.layers.Dense(30, activation = 'relu', input_shape = x_train.shape[1:]),
#     keras.layers.Dense(1),
# ])

input = keras.layers.Input(shape=x_train.shape[1:])
#函數式API 這個層次可以像函數一樣地去使用它
hidden1 = keras.layers.Dense(30, activation = 'relu')(input)
hidden2 = keras.layers.Dense(30,activation = 'relu')(hidden1)
#wide模型
#input wide模型的輸入 hidden2 deep模型的輸出
concat = keras.layers.concatenate([input,hidden2])
output = keras.layers.Dense(1)(concat)
#使用model把模型固定下來
model = keras.models.Model(inputs=[input],outputs=[output])

model.summary()
model.compile(loss = "mean_squared_error", optimizer = "adam")

callbacks = [keras.callbacks.EarlyStopping(patience = 5, min_delta = 1e-2)]
history = model.fit(x_train_scaled, y_train, validation_data = (x_valid_scaled, y_valid), epochs = 100, callbacks = callbacks)

2、子類API實現

單輸入單輸出

#子類API實現wide_deep模型
class WideDeepModel(keras.models.Model):
    def __init__(self):#初始化函數
        super(WideDeepModel,self).__init__()
        '''定義模型的層次'''
        self.hidden1_layer = keras.layers.Dense(30,activation='relu')
        self.hidden2_layer = keras.layers.Dense(30,activation='relu')
        self.output_layer = keras.layers.Dense(1)
    def call(self,input):
        '''
        重載此函數 完成模型的正向計算
        '''
        hidden1 = self.hidden1_layer(input)
        hidden2 = self.hidden2_layer(hidden1)
        concat = keras.layers.concatenate([input,hidden2])
        output = self.output_layer(concat)
        return output
#model = WideDeepModel()
model = keras.models.Sequential([
    WideDeepModel(),
])
model.build(input_shape=(None,8))
        
model.summary()
model.compile(loss = "mean_squared_error", optimizer = "adam")

callbacks = [keras.callbacks.EarlyStopping(patience = 5, min_delta = 1e-2)]
history = model.fit(x_train_scaled, y_train, validation_data = (x_valid_scaled, y_valid), epochs = 100, callbacks = callbacks)

3、多輸入單輸出

# 函數式API實現wide_deep模型
#多輸入 前5個特徵爲wide模型的輸入 後6個特徵爲deep模型的輸入
input_wide = keras.layers.Input(shape=[5])
input_deep = keras.layers.Input(shape=[6])
hidden1 = keras.layers.Dense(30,activation = 'relu')(input_deep)
hidden2 = keras.layers.Dense(30,activation = 'relu')(hidden1)
concat = keras.layers.concatenate([input_wide,hidden2])
output = keras.layers.Dense(1)(concat)
model = keras.models.Model(inputs = [input_wide,input_deep],outputs=[output])

        
model.summary()
model.compile(loss = "mean_squared_error", optimizer = "adam")

callbacks = [keras.callbacks.EarlyStopping(patience = 5, min_delta = 1e-2)]
x_train_scaled_wide = x_train_scaled[:,:5]#wide取前5個feature
x_train_scaled_deep = x_train_scaled[:,2:]#deep取後6個feature
x_valid_scaled_wide = x_valid_scaled[:,:5]
x_valid_scaled_deep = x_valid_scaled[:,2:]
x_test_scaled_wide = x_test_scaled[:,:5]
x_test_scaled_deep = x_test_scaled[:,2:]
history = model.fit([x_train_scaled_wide,x_train_scaled_deep], y_train, validation_data = ([x_valid_scaled_wide,x_valid_scaled_deep], y_valid), epochs = 100, callbacks = callbacks)

4、多輸入多輸出

    
input_wide = keras.layers.Input(shape=[5])
input_deep = keras.layers.Input(shape=[6])
hidden1 = keras.layers.Dense(30,activation = 'relu')(input_deep)
hidden2 = keras.layers.Dense(30,activation = 'relu')(hidden1)
concat = keras.layers.concatenate([input_wide,hidden2])
output = keras.layers.Dense(1)(concat)
output2 = keras.layers.Dense(1)(hidden2)
model = keras.models.Model(inputs = [input_wide,input_deep],outputs=[output,output2])

        
model.summary()
model.compile(loss = "mean_squared_error", optimizer = "adam")

callbacks = [keras.callbacks.EarlyStopping(patience = 5, min_delta = 1e-2)]
x_train_scaled_wide = x_train_scaled[:,:5]#wide取前5個feature
x_train_scaled_deep = x_train_scaled[:,2:]#deep取後6個feature
x_valid_scaled_wide = x_valid_scaled[:,:5]
x_valid_scaled_deep = x_valid_scaled[:,2:]
x_test_scaled_wide = x_test_scaled[:,:5]
x_test_scaled_deep = x_test_scaled[:,2:]
history = model.fit([x_train_scaled_wide,x_train_scaled_deep],[y_train, y_train], validation_data = ([x_valid_scaled_wide,x_valid_scaled_deep], [y_valid,y_valid]), epochs = 100, callbacks = callbacks)

2、超參數搜索

1、手動實現超參數搜索

根據圖像選取較好的超參數

#手動實現learning_rate的搜索[1e-4,3e-4,1e-3,3e-3,1e-2,3e-2]
#W = W + grad* learning_rate
learning_rate = [1e-4,3e-4,1e-3,3e-3,1e-2,3e-2]
# 對每個參數都需要定義模型
histories = []
for lr in learning_rate:
    model = keras.models.Sequential([
        keras.layers.Dense(30, activation = 'relu', input_shape = x_train.shape[1:]),
        keras.layers.Dense(1),
    ])
    optimizer = keras.optimizers.SGD(lr)
    
    model.compile(loss = "mean_squared_error", optimizer = optimizer)
    callbacks = [keras.callbacks.EarlyStopping(patience = 5, min_delta = 1e-2)]
    history = model.fit(x_train_scaled, y_train, validation_data = (x_valid_scaled, y_valid), epochs = 100, callbacks = callbacks)
    histories.append(history)

def plot_learning_curves(history):
    pd.DataFrame(history.history).plot(figsize=(8,5))
    plt.grid(True)
    plt.gca().set_ylim(0,1)
    plt.show()

for lr,history in zip(learning_rate,histories):
    print("learning rate:",lr)
    plot_learning_curves(history)

2、利用sklearn庫實現自動超參數搜索

RandomizedSearchCV實現超參數搜索 分爲三步：
1、tf.keras.model轉化爲sklearn的model 利用tf.keras.wrappers.scikit_learn.KerasRegressor

def build_model(hidden_layers = 1,#中間層數
                layer_size =30,#中間層大小
                learning_rate=1e-3):#學習率
    model = keras.models.Sequential()
    model.add(keras.layers.Dense(layer_size, activation = 'relu', input_shape = x_train.shape[1:]))
    for _ in range(hidden_layers-1):
        model.add(keras.layers.Dense(layer_size, activation = 'relu'))
    model.add(keras.layers.Dense(1))
    optimizer = keras.optimizers.SGD(learning_rate)
    model.compile(loss = "mean_squared_error", optimizer = optimizer)#  mean_squared_error == mse
    return model
sklearn_model = keras.wrappers.scikit_learn.KerasRegressor(build_model)
callbacks = [keras.callbacks.EarlyStopping(patience = 5, min_delta = 1e-2)]
history = sklearn_model.fit(x_train_scaled,y_train,
                            epochs = 100,
                            validation_data = (x_valid_scaled,y_valid),
                            callbacks = callbacks)

2、定義參數集合

以上完成了第一步
下面開始第二步，進行參數集合的定義
要搜索的參數爲hidden_layers = 1、layer_size =30、learning_rate=1e-3

from scipy.stats import reciprocal
# f(x) = 1/(x*log(b/a)) a<=x<=b f(x)爲概率 reciprocal的分佈函數
param_distribution = {
    "hidden_layers":[1,2,3,4],
    "layer_size":np.arange(1,100),
    #希望learning_rate爲連續取值 定義區間即可
    "learning_rate":reciprocal(1e-4,1e-2),
}

3、使用RandomizedSearchCV實現超參數搜索

from sklearn.model_selection import RandomizedSearchCV

random_search_cv = RandomizedSearchCV(sklearn_model,
                                      param_distribution,
                                     n_iter = 10,#從param_distribution散播出來的參數集合的數目
                                      #cv = 3,通過此修改cross_validation的n
                                     n_jobs =1)#並行處理的數目
random_search_cv.fit(x_train_scaled,y_train,epochs = 100,
                        validation_data = (x_valid_scaled,y_valid),
                        callbacks = callbacks)
#進行超參數搜索時 使用了 cross_validation的機制
#cross_validation：訓練集分爲n-1份 1份用於驗證
#超參搜索完之後再用所有數據訓練一次

最後，對得到的模型進行評估

print(random_search_cv.best_params_)#最好的參數
print(random_search_cv.best_score_)#最好的值
print(random_search_cv.best_estimator_)#最好的model

model = random_search_cv.best_estimator_.model
model.evaluate(x_test_scaled, y_test)