一、簡述算法線性LDA、k-means和SVM算法
算法
基本思想
LDA
LDA 是一種可作爲特徵抽取的技術,其目標是向最大化類間差異,最小化類內差異的方向投影,以利於分類等任務即將不同類的樣本有效的分開。LDA 可以提高數據分析過程中的計算效率,對於未能正則化的模型,可以降低維度災難帶來的過擬合
k-means
在K-means算法中K是事先給定的,這個K值的選定是非常難以估計的;在K-means算法中,首先需要根據初始聚類中心來確定一個初始劃分,然後對初始劃分進行優化;從K-means算法框架可以看出,該算法需要不斷地進行樣本分類調整,不斷地計算調整後的新的聚類中心,因此當數據量非常大時,算法的時間開銷是非常大的。
SVM
SVM (支持向量機)是一種算法,它以以下方法做工,它是一種種非線性映射額,把原訓練數據映射到較高的維上,在新的維上,它搜索最佳分離超平面。
二、線性判別分析LDA
1.鳶尾花數據集
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets.samples_generator import make_classification
class LDA():
def Train(self, X, y):
"""X爲訓練數據集,y爲訓練label"""
X1 = np.array([X[i] for i in range(len(X)) if y[i] == 0])
X2 = np.array([X[i] for i in range(len(X)) if y[i] == 1])
# 求中心點
mju1 = np.mean(X1, axis=0) # mju1是ndrray類型
mju2 = np.mean(X2, axis=0)
# dot(a, b, out=None) 計算矩陣乘法
cov1 = np.dot((X1 - mju1).T, (X1 - mju1))
cov2 = np.dot((X2 - mju2).T, (X2 - mju2))
Sw = cov1 + cov2
# 計算w
w = np.dot(np.mat(Sw).I, (mju1 - mju2).reshape((len(mju1), 1)))
# 記錄訓練結果
self.mju1 = mju1 # 第1類的分類中心
self.cov1 = cov1
self.mju2 = mju2 # 第2類的分類中心
self.cov2 = cov2
self.Sw = Sw # 類內散度矩陣
self.w = w # 判別權重矩陣
def Test(self, X, y):
"""X爲測試數據集,y爲測試label"""
# 分類結果
y_new = np.dot((X), self.w)
# 計算fisher線性判別式
nums = len(y)
c1 = np.dot((self.mju1 - self.mju2).reshape(1, (len(self.mju1))), np.mat(self.Sw).I)
c2 = np.dot(c1, (self.mju1 + self.mju2).reshape((len(self.mju1), 1)))
c = 1/2 * c2 # 2個分類的中心
h = y_new - c
# 判別
y_hat = []
for i in range(nums):
if h[i] >= 0:
y_hat.append(0)
else:
y_hat.append(1)
# 計算分類精度
count = 0
for i in range(nums):
if y_hat[i] == y[i]:
count += 1
precise = count / nums
# 顯示信息
print("測試樣本數量:", nums)
print("預測正確樣本的數量:", count)
print("測試準確度:", precise)
return precise
if '__main__' == __name__:
# 產生分類數據
n_samples = 500
X, y = make_classification(n_samples=n_samples, n_features=2, n_redundant=0, n_classes=2,n_informative=1, n_clusters_per_class=1, class_sep=0.5, random_state=10)
# LDA線性判別分析(二分類)
lda = LDA()
# 60% 用作訓練,40%用作測試
Xtrain = X[:299, :]
Ytrain = y[:299]
Xtest = X[300:, :]
Ytest = y[300:]
lda.Train(Xtrain, Ytrain)
precise = lda.Test(Xtest, Ytest)
# 原始數據
plt.scatter(X[:, 0], X[:, 1], marker='o', c=y)
plt.xlabel("x1")
plt.ylabel("x2")
plt.title("Test precise:" + str(precise))
plt.show()
2.月亮數據集
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_moons
class LDA():
def Train(self, X, y):
"""X爲訓練數據集,y爲訓練label"""
X1 = np.array([X[i] for i in range(len(X)) if y[i] == 0])
X2 = np.array([X[i] for i in range(len(X)) if y[i] == 1])
# 求中心點
mju1 = np.mean(X1, axis=0) # mju1是ndrray類型
mju2 = np.mean(X2, axis=0)
# dot(a, b, out=None) 計算矩陣乘法
cov1 = np.dot((X1 - mju1).T, (X1 - mju1))
cov2 = np.dot((X2 - mju2).T, (X2 - mju2))
Sw = cov1 + cov2
# 計算w
w = np.dot(np.mat(Sw).I, (mju1 - mju2).reshape((len(mju1), 1)))
# 記錄訓練結果
self.mju1 = mju1 # 第1類的分類中心
self.cov1 = cov1
self.mju2 = mju2 # 第1類的分類中心
self.cov2 = cov2
self.Sw = Sw # 類內散度矩陣
self.w = w # 判別權重矩陣
def Test(self, X, y):
"""X爲測試數據集,y爲測試label"""
# 分類結果
y_new = np.dot((X), self.w)
# 計算fisher線性判別式
nums = len(y)
c1 = np.dot((self.mju1 - self.mju2).reshape(1, (len(self.mju1))), np.mat(self.Sw).I)
c2 = np.dot(c1, (self.mju1 + self.mju2).reshape((len(self.mju1), 1)))
c = 1/2 * c2 # 2個分類的中心
h = y_new - c
# 判別
y_hat = []
for i in range(nums):
if h[i] >= 0:
y_hat.append(0)
else:
y_hat.append(1)
# 計算分類精度
count = 0
for i in range(nums):
if y_hat[i] == y[i]:
count += 1
precise = count / (nums+0.000001)
# 顯示信息
print("測試樣本數量:", nums)
print("預測正確樣本的數量:", count)
print("測試準確度:", precise)
return precise
if '__main__' == __name__:
# 產生分類數據
X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
# LDA線性判別分析(二分類)
lda = LDA()
# 60% 用作訓練,40%用作測試
Xtrain = X[:60, :]
Ytrain = y[:60]
Xtest = X[40:, :]
Ytest = y[40:]
lda.Train(Xtrain, Ytrain)
precise = lda.Test(Xtest, Ytest)
# 原始數據
plt.scatter(X[:, 0], X[:, 1], marker='o', c=y)
plt.xlabel("x1")
plt.ylabel("x2")
plt.title("Test precise:" + str(precise))
plt.show()
三、k-means聚類分析
1.鳶尾花數據集
import matplotlib.pyplot as plt
import numpy as np
from sklearn.cluster import KMeans
from sklearn import datasets
from sklearn.datasets import load_iris
iris = load_iris()
X = iris.data[:] ##表示我們只取特徵空間中的後兩個維度
estimator = KMeans(n_clusters=5)#構造聚類器
estimator.fit(X)#聚類
label_pred = estimator.labels_ #獲取聚類標籤
#繪製k-means結果
x0 = X[label_pred == 0]
x1 = X[label_pred == 1]
x2 = X[label_pred == 2]
x3 = X[label_pred == 3]
plt.scatter(x0[:, 0], x0[:, 1], c = "red", marker='o', label='label0')
plt.scatter(x1[:, 0], x1[:, 1], c = "green", marker='*', label='label1')
#plt.scatter(x2[:, 0], x2[:, 1], c = "blue", marker='+', label='label2')
#plt.scatter(x3[:, 0], x3[:, 1], c = "yellow", marker='o', label='label3')
plt.xlabel('petal length')
plt.ylabel('petal width')
plt.legend(loc=2)
plt.show()
2.月亮數據集
import matplotlib.pyplot as plt
import numpy as np
from sklearn.cluster import KMeans
from sklearn.datasets import make_moons
X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
estimator = KMeans(n_clusters=5)#構造聚類器
estimator.fit(X)#聚類
label_pred = estimator.labels_ #獲取聚類標籤
#繪製k-means結果
x0 = X[label_pred == 0]
x1 = X[label_pred == 1]
x2 = X[label_pred == 2]
x3 = X[label_pred == 3]
plt.scatter(x0[:, 0], x0[:, 1], c = "red", marker='o', label='label0')
plt.scatter(x1[:, 0], x1[:, 1], c = "green", marker='*', label='label1')
#plt.scatter(x2[:, 0], x2[:, 1], c = "blue", marker='+', label='label2')
#plt.scatter(x3[:, 0], x3[:, 1], c = "yellow", marker='o', label='label3')
plt.xlabel('petal length')
plt.ylabel('petal width')
plt.legend(loc=2)
plt.show()
四、SVM(支持向量機)算法
1.鳶尾花數據集
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets, svm
import pandas as pd
from pylab import *
mpl.rcParams['font.sans-serif'] = ['SimHei']
iris = datasets.load_iris()
iris = datasets.load_iris()
X = iris.data
y = iris.target
X = X[y != 0, :2] # 選擇X的前兩個特性
y = y[y != 0]
n_sample = len(X)
np.random.seed(0)
order = np.random.permutation(n_sample) # 排列,置換
X = X[order]
y = y[order].astype(np.float)
X_train = X[:int(.9 * n_sample)]
y_train = y[:int(.9 * n_sample)]
X_test = X[int(.9 * n_sample):]
y_test = y[int(.9 * n_sample):]
#合適的模型
for fig_num, kernel in enumerate(('linear', 'rbf','poly')): # 徑向基函數 (Radial Basis Function 簡稱 RBF),常用的是高斯基函數
clf = svm.SVC(kernel=kernel, gamma=10) # gamma是“rbf”、“poly”和“sigmoid”的核係數。
clf.fit(X_train, y_train)
plt.figure(str(kernel))
plt.xlabel('x1')
plt.ylabel('x2')
plt.scatter(X[:, 0], X[:, 1], c=y, zorder=10, cmap=plt.cm.Paired, edgecolor='k', s=20)
# zorder: z方向上排列順序,數值越大,在上方顯示
# paired兩個色彩相近輸出(paired)
# 圈出測試數據
plt.scatter(X_test[:, 0], X_test[:, 1], s=80, facecolors='none',zorder=10, edgecolor='k')
plt.axis('tight') #更改 x 和 y 軸限制,以便顯示所有數據
x_min = X[:, 0].min()
x_max = X[:, 0].max()
y_min = X[:, 1].min()
y_max = X[:, 1].max()
XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]
Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()]) # 樣本X到分離超平面的距離
Z = Z.reshape(XX.shape)
plt.contourf(XX,YY,Z>0,cmap=plt.cm.Paired)
plt.contour(XX, YY, Z, colors=['r', 'k', 'b'],
linestyles=['--', '-', '--'], levels=[-0.5, 0, 0.5]) # 範圍
plt.title(kernel)
plt.show()
2.月亮數據集
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
import numpy as np
import matplotlib as mpl
from sklearn.datasets import make_moons
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
# 爲了顯示中文
mpl.rcParams['font.sans-serif'] = [u'SimHei']
mpl.rcParams['axes.unicode_minus'] = False
X, y = make_moons(n_samples=100, noise=0.15, random_state=42)
def plot_dataset(X, y, axes):
plt.plot(X[:, 0][y==0], X[:, 1][y==0], "bs")
plt.plot(X[:, 0][y==1], X[:, 1][y==1], "g^")
plt.axis(axes)
plt.grid(True, which='both')
plt.xlabel(r"$x_1$", fontsize=20)
plt.ylabel(r"$x_2$", fontsize=20, rotation=0)
plt.title("月亮數據",fontsize=20)
plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
plt.show()
polynomial_svm_clf = Pipeline([
# 將源數據 映射到 3階多項式
("poly_features", PolynomialFeatures(degree=3)),
# 標準化
("scaler", StandardScaler()),
# SVC線性分類器
("svm_clf", LinearSVC(C=10, loss="hinge", random_state=42))
])
polynomial_svm_clf.fit(X, y)
def plot_predictions(clf, axes):
# 打表
x0s = np.linspace(axes[0], axes[1], 100)
x1s = np.linspace(axes[2], axes[3], 100)
x0, x1 = np.meshgrid(x0s, x1s)
X = np.c_[x0.ravel(), x1.ravel()]
y_pred = clf.predict(X).reshape(x0.shape)
y_decision = clf.decision_function(X).reshape(x0.shape)
# print(y_pred)
# print(y_decision)
plt.contourf(x0, x1, y_pred, cmap=plt.cm.brg, alpha=0.2)
plt.contourf(x0, x1, y_decision, cmap=plt.cm.brg, alpha=0.1)
plot_predictions(polynomial_svm_clf, [-1.5, 2.5, -1, 1.5])
plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
plt.show()
from sklearn.svm import SVC
gamma1, gamma2 = 0.1, 5
C1, C2 = 0.001, 1000
hyperparams = (gamma1, C1), (gamma1, C2)
svm_clfs = []
for gamma, C in hyperparams:
rbf_kernel_svm_clf = Pipeline([
("scaler", StandardScaler()),
("svm_clf", SVC(kernel="rbf", gamma=gamma, C=C))
])
rbf_kernel_svm_clf.fit(X, y)
svm_clfs.append(rbf_kernel_svm_clf)
plt.figure(figsize=(11, 7))
for i, svm_clf in enumerate(svm_clfs):
plt.subplot(221 + i)
plot_predictions(svm_clf, [-1.5, 2.5, -1, 1.5])
plot_dataset(X, y, [-1.5, 2.5, -1, 1.5])
gamma, C = hyperparams[i]
plt.title(r"$\gamma = {}, C = {}$".format(gamma, C), fontsize=16)
plt.tight_layout()
plt.show()
五、SVM算法的優缺點
優點
①SVM算法既可以解決線性問題,又可以解決非線性問題②非線性映射是SVM方法的理論基礎,SVM利用內積核函數代替向高維空間的非線性映射③對特徵空間劃分的最優超平面是SVM的目標,最大化分類邊際的思想是SVM方法的核心④支持向量是SVM的訓練結果,在SVM分類決策中起決定作用的是支持向量
優點
①對參數調節和核函數的選擇敏感②不易處理多分類問題③對大規模訓練樣本難以實施④SVM的可解釋性較差,無法給出決策樹那樣的規則
周志華《機器學習》LDA深入補充推導和python實現(二分類問題)