機器學習系列（七） 多項式迴歸和模型泛化（學習曲線、交叉驗證、正則化） 2020.6.8

前言

本節學習多項式迴歸和模型泛化

• 學習曲線
• 交叉驗證
• 正則化

1、多項式迴歸

• 相比線性迴歸，爲原數據添加新特徵
• 新特徵是對原數據的多項式組合
• 升維處理，在SVM裏會體現

實現如下

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression

"""多項式迴歸"""
# 數據
x = np.random.uniform(-3, 3, size=100)
X = x.reshape(-1, 1)
y = 0.5 * x**2 + x + 2 + np.random.normal(0, 1, 100)
plt.scatter(x, y)
plt.show()
# 添加一個特徵
X2 = np.hstack([X, X**2])
print(X2.shape)
# 對新數據集線性迴歸
lin_reg2 = LinearRegression()
lin_reg2.fit(X2, y)
y_predict2 = lin_reg2.predict(X2)
plt.scatter(x, y)
plt.plot(np.sort(x), y_predict2[np.argsort(x)], color='r') #平滑曲線需要對x排序
plt.show()
print(lin_reg2.coef_)
print(lin_reg2.intercept_)

使用scikit庫實現如下

import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression

"""用scikit實現多項式迴歸"""
# 數據
x = np.random.uniform(-3, 3, size=100)
X = x.reshape(-1, 1)
y = 0.5 * x**2 + x + 2 + np.random.normal(0, 1, 100)
# 使用scikit增加特徵
poly = PolynomialFeatures(degree=2) #初始化，兩個維度
poly.fit(X)
X2 = poly.transform(X)
print(X2.shape)
print(X2[:5,:]) #可以看到第一列是1，第二列是X，第三列是X^2
# 線性迴歸
lin_reg2 = LinearRegression()
lin_reg2.fit(X2, y)
y_predict2 = lin_reg2.predict(X2)
plt.scatter(x, y)
plt.plot(np.sort(x), y_predict2[np.argsort(x)], color='r')
plt.show()
print(lin_reg2.coef_)
print(lin_reg2.intercept_)

# 可以用pipeline一次性完成
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
poly_reg = Pipeline([
    ("poly", PolynomialFeatures(degree=2)), #添加維度
    ("std_scaler", StandardScaler()), #歸一化
    ("lin_reg", LinearRegression()) #線性迴歸
])
poly_reg.fit(X, y)
y_predict = poly_reg.predict(X)
plt.scatter(x, y)
plt.plot(np.sort(x), y_predict[np.argsort(x)], color='r')
plt.show()

最終效果如下

2、學習曲線

主要是一個模型泛化問題
也就是過擬合和欠擬合的問題
看圖很好理解

學習曲線
隨着樣本增多，算法訓練出的模型的表現能力

實現如下

import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

""" 學習曲線 """
# 數據
np.random.seed(666)
x = np.random.uniform(-3.0, 3.0, size=100)
X = x.reshape(-1, 1)
y = 0.5 * x**2 + x + 2 + np.random.normal(0, 1, size=100)
plt.scatter(x, y)
plt.show()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=10)
print(X_train.shape)

# 學習曲線
train_score = []
test_score = []
for i in range(1, 76):
    lin_reg = LinearRegression()
    lin_reg.fit(X_train[:i], y_train[:i])
    y_train_predict = lin_reg.predict(X_train[:i])
    train_score.append(mean_squared_error(y_train[:i], y_train_predict))
    y_test_predict = lin_reg.predict(X_test)
    test_score.append(mean_squared_error(y_test, y_test_predict))
plt.plot([i for i in range(1, 76)], np.sqrt(train_score), label="train")
plt.plot([i for i in range(1, 76)], np.sqrt(test_score), label="test")
plt.legend()
plt.show()

"""封裝成函數"""
def plot_learning_curve(algo, X_train, X_test, y_train, y_test):
    train_score = []
    test_score = []
    for i in range(1, len(X_train) + 1):
        algo.fit(X_train[:i], y_train[:i])
        y_train_predict = algo.predict(X_train[:i])
        train_score.append(mean_squared_error(y_train[:i], y_train_predict))
        y_test_predict = algo.predict(X_test)
        test_score.append(mean_squared_error(y_test, y_test_predict))
    plt.plot([i for i in range(1, len(X_train) + 1)],
             np.sqrt(train_score), label="train")
    plt.plot([i for i in range(1, len(X_train) + 1)],
             np.sqrt(test_score), label="test")
    plt.legend()
    plt.axis([0, len(X_train) + 1, 0, 4]) #座標軸範圍作了限定
    plt.show()
plot_learning_curve(LinearRegression(), X_train, X_test, y_train, y_test)

"""在多項式迴歸中的學習曲線"""
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
def PolynomialRegression(degree):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("lin_reg", LinearRegression())
    ])
poly2_reg = PolynomialRegression(degree=2)
plot_learning_curve(poly2_reg, X_train, X_test, y_train, y_test)
poly20_reg = PolynomialRegression(degree=20)
plot_learning_curve(poly20_reg, X_train, X_test, y_train, y_test)

通過學習曲線可以判斷欠擬合還是過擬合
欠擬合、最佳、過擬合的效果如下

• 欠擬合：比起最佳的，穩定值會高
• 過擬合：train的差別不大，test的誤差大，且離得遠

3、交叉驗證

之前劃分train-test數據集的做法，可能使得模型針對test數據集過擬合
解決辦法就是交叉驗證

• 增加驗證數據集
• train數據集分爲k份，每次訓練k-1份留1份驗證
• k個模型結果平均作爲評判

實現如下

import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier

"""交叉驗證"""
# 數據
digits = datasets.load_digits()
X = digits.data
y = digits.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=666)
# kNN下原先的train-test方式調參
best_k, best_p, best_score = 0, 0, 0
for k in range(2, 11):
    for p in range(1, 6):
        knn_clf = KNeighborsClassifier(weights="distance", n_neighbors=k, p=p)
        knn_clf.fit(X_train, y_train)
        score = knn_clf.score(X_test, y_test)
        if score > best_score:
            best_k, best_p, best_score = k, p, score
print("Best K =", best_k)
print("Best P =", best_p)
print("Best Score =", best_score)
best_knn_clf = KNeighborsClassifier(weights="distance", n_neighbors=best_k, p=best_p)
best_knn_clf.fit(X_train, y_train)
print(best_knn_clf.score(X_test, y_test))
# kNN下交叉驗證方式調參
from sklearn.model_selection import cross_val_score
knn_clf = KNeighborsClassifier()
print(cross_val_score(knn_clf, X_train, y_train))
best_k, best_p, best_score = 0, 0, 0
for k in range(2, 11):
    for p in range(1, 6):
        knn_clf = KNeighborsClassifier(weights="distance", n_neighbors=k, p=p)
        scores = cross_val_score(knn_clf, X_train, y_train, cv=3) #使用交叉驗證來調參
        score = np.mean(scores)
        if score > best_score:
            best_k, best_p, best_score = k, p, score
print("Best K =", best_k)
print("Best P =", best_p)
print("Best Score =", best_score)
best_knn_clf = KNeighborsClassifier(weights="distance", n_neighbors=best_k, p=best_p)
best_knn_clf.fit(X_train, y_train)
print(best_knn_clf.score(X_test, y_test))
# 前面學習的網格搜索已經用了交叉驗證

4、正則化

主要是以下幾項內容

• 嶺迴歸：L2正則項

• lasso迴歸：L1正則項

• 彈性網：用r表示L1正則項和L2正則項的比例

實現如下

import numpy as np
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split

"""模型正則化"""
# 數據
np.random.seed(42)
x = np.random.uniform(-3.0, 3.0, size=100)
X = x.reshape(-1, 1)
y = 0.5 * x + 3 + np.random.normal(0, 1, size=100)
plt.scatter(x, y)
plt.show()
# 多項式迴歸
def PolynomialRegression(degree):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("lin_reg", LinearRegression())
    ])
# 繪圖
def plot_model(model):
    X_plot = np.linspace(-3, 3, 100).reshape(100, 1)
    y_plot = model.predict(X_plot)
    plt.scatter(x, y)
    plt.plot(X_plot[:,0], y_plot, color='r')
    plt.axis([-3, 3, 0, 6])
    plt.show()
# 原實現
np.random.seed(666)
X_train, X_test, y_train, y_test = train_test_split(X, y)
poly_reg = PolynomialRegression(degree=20)
poly_reg.fit(X_train, y_train)
y_poly_predict = poly_reg.predict(X_test)
print(mean_squared_error(y_test, y_poly_predict))
plot_model(poly_reg)

# 正則化
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso

# 嶺迴歸
def RidgeRegression(degree, alpha):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("ridge_reg", Ridge(alpha=alpha))
    ])
# α= 0.0001
ridge1_reg = RidgeRegression(20, 0.0001)
ridge1_reg.fit(X_train, y_train)
y1_predict = ridge1_reg.predict(X_test)
print(mean_squared_error(y_test, y1_predict))
plot_model(ridge1_reg)
# α= 1
ridge2_reg = RidgeRegression(20, 1)
ridge2_reg.fit(X_train, y_train)
y2_predict = ridge2_reg.predict(X_test)
print(mean_squared_error(y_test, y2_predict))
plot_model(ridge2_reg)
# α= 100
ridge3_reg = RidgeRegression(20, 100)
ridge3_reg.fit(X_train, y_train)
y3_predict = ridge3_reg.predict(X_test)
print(mean_squared_error(y_test, y3_predict))
plot_model(ridge3_reg)

# lasso迴歸
def LassoRegression(degree, alpha):
    return Pipeline([
        ("poly", PolynomialFeatures(degree=degree)),
        ("std_scaler", StandardScaler()),
        ("lasso_reg", Lasso(alpha=alpha))
    ])
# α= 0.01
lasso1_reg = LassoRegression(20, 0.01)
lasso1_reg.fit(X_train, y_train)
y1_predict = lasso1_reg.predict(X_test)
print(mean_squared_error(y_test, y1_predict))
plot_model(lasso1_reg)
# α= 0.1
lasso2_reg = LassoRegression(20, 0.1)
lasso2_reg.fit(X_train, y_train)
y2_predict = lasso2_reg.predict(X_test)
print(mean_squared_error(y_test, y2_predict))
plot_model(lasso2_reg)
# α= 1
lasso3_reg = LassoRegression(20, 1)
lasso3_reg.fit(X_train, y_train)
y3_predict = lasso3_reg.predict(X_test)
print(mean_squared_error(y_test, y3_predict))
plot_model(lasso3_reg)

嶺迴歸較好效果

lasso迴歸較好效果

結語

本節學習了多項式迴歸
以及各種模型泛化的的考慮