支持向量機通俗導論——學習（二）

優化後的SMO算法：

步驟1：先“掃描”所有乘子，把第一個違反KKT條件的作爲更新對象，令爲a1；
步驟2：在所有不違反KKT條件的乘子中，選擇使|E1 −E2|最大的a2進行更新，使得能最大限度增大目標函數的值（類似於梯度下降. 此外，而，求出來的E代表函數ui對輸入xi的預測值與真實輸出類標記yi之差）。
最後，每次更新完兩個乘子的優化後，都需要再重新計算b，及對應的Ei值。
————————————————
版權聲明：本文爲CSDN博主「v_JULY_v」的原創文章，遵循 CC 4.0 BY-SA 版權協議，轉載請附上原文出處鏈接及本聲明。
原文鏈接：https://blog.csdn.net/v_JULY_v/article/details/7624837

MySVM——smop

# -*- coding: utf-8 -*-
import random
import numpy as np
import matplotlib.pyplot as plt

#加載訓練集
def loadDataSet(filename):
    dataMat = []
    labelMat = []
    fr = open(filename)
    for line in fr.readlines():
        lineArr = line.strip().split('\t')
        #s.strip(rm) 刪除s字符串中開頭、結尾處，位於rm刪除序列的字符;split按括號內的進行返回一個列表
        dataMat.append([float(lineArr[0]),float(lineArr[1])])
        labelMat.append(float(lineArr[2]))
    return dataMat,labelMat
# 定義了一個類作爲數據結構來存儲一些信息;建立一個類存放基本數據以及alphas的緩存
class optStruct:
    def __init__(self,dataMatIn, classLabels, C, toler, kTup):
        self.X = dataMatIn             #數據
        self.labelMat = classLabels    #標籤
        self.C = C                     #容忍度
        self.tol = toler               #誤差容忍度
        self.m = np.shape(dataMatIn)[0]#數據個數，給出行；1的話是給出列
        self.alphas = np.mat(np.zeros((self.m,1))) #alpha值 (m,1)
        self.b = 0                     #常數項
        self.eCache = np.mat(np.zeros((self.m,2)))#保存誤差和下表 (m,2)

#輔助函數一: 創建一個樣本點返回來的隨機整數（0，m）,即選擇alpha（i，j）
def selectJrand(i,m):
    j = i
    while (j == i):
        j  = int(random.uniform(0,m)) #返回一個浮點數 N，取值範圍爲0,m
    return j
# 輔助函數二：將aj規劃到[0,C]範圍
def ClipAlpha(aj,H,L):
    if aj > H:
        aj = H
    if aj < L:
        aj = L
    return aj
# 建立計算錯誤率的函數
def calcEk(oS, k):
    fxk = float(np.multiply(oS.alphas, oS.labelMat).T *\
          (oS.X*oS.X[k,:].T) +oS.b)
    Ek = fxk - float(oS.labelMat[k])
    return Ek
# 啓發式找j:選擇最長步長的那個（就是選擇|Ei-Ej|最大）
def selectJ(i, oS, Ei):
    maxK = -1
    maxDeltaE = 0
    Ej = 0
    oS.eCache[i] = [1,Ei] #誤差緩存。1st 列爲1時表示有效（計算好了誤差）
    validEcachelist = np.nonzero(oS.eCache[:,0].A)[0] #1\A.將matrix類型轉換成array類型2\返回數組中不爲0的元素下標
    if(len(validEcachelist )) > 1:
        for k in validEcachelist:
            if k == i:
                continue
            Ek = calcEk(oS,k)
            deltaE = abs(Ei - Ek)
            if (deltaE > maxDeltaE):
                maxK = k
                maxDeltaE = deltaE
                Ej = Ek
        return maxK, Ej
    else:
        j = selectJrand(i, oS.m)
        Ej = calcEk(oS,j)
    return j, Ej
#更新錯誤率Ek
def updateEk(oS,k):
    Ek = calcEk(oS,k)
    oS.eCache[k] = [1,Ek]
# 更新b並且返回兩個alpha參數是否改變的情況
def innerL(i,oS):
    Ei = calcEk(oS, i)
    if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or\
    ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):#不滿足KKT
        j,Ej = selectJ(i, oS, Ei)  # 尋找|Ei-Ej|最大的j
        alphaIold = oS.alphas[i].copy()
        alphaJold = oS.alphas[j].copy()
        if (oS.labelMat[i] != oS.labelMat[j]):
            L = max(0, oS.alphas[j] - oS.alphas[i])
            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])
        else:
            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)
            H = min(oS.C, oS.alphas[j] + oS.alphas[i])
        if L==H:
            print("L==H")
            return 0
        eta = 2.0 * oS.X[i,:]*oS.X[j,:].T-oS.X[i,:]*oS.X[i,:].T-oS.X[j,:]*oS.X[j,:].T
        if eta >= 0:
            print("eta>=0")
            return 0
        oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/eta
        oS.alphas[j] = ClipAlpha(oS.alphas[j],H,L)
        updateEk(oS, j)
        if (abs(oS.alphas[j] - alphaJold) < oS.tol):
            print("j not moving enough")
            return 0
        oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])
        updateEk(oS, i)
        b1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.X[i,:]*oS.X[i,:].T - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.X[i,:]*oS.X[j,:].T
        b2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.X[i,:]*oS.X[j,:].T - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.X[j,:]*oS.X[j,:].T
        if (0 < oS.alphas[i] < oS.C):
            oS.b = b1
        elif (0 < oS.alphas[j] < oS.C):
            oS.b = b2
        else:
            oS.b = (b1 + b2)/2.0
        return 1
    else:
        return 0
# 計算w
def calcWs(dataMat, labelMat, alphas):
    alphas, dataMat, labelMat = np.array(alphas), np.array(dataMat), np.array(labelMat)
    # np.tile：np.tile(a,(1,2))第一個參數爲Y軸擴大倍數，第二個爲X軸擴大倍數，即原矩陣列數*2
    w = np.dot((np.tile(labelMat.reshape(1, -1).T, (1, 2)) * dataMat).T, alphas)  #得到2*1
    return w.tolist()  # 數組——>列表

def smoP(dataMatIn, classLabels, C, toler, maxIter,kTup=('lin', 0)):
    oS = optStruct(np.mat(dataMatIn),np.mat(classLabels).transpose(),C,toler, kTup)
    iter = 0
    entireSet = True
    alphaPairsChanged = 0
    while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):
        alphaPairsChanged = 0
        if entireSet:
            for i in range(oS.m):
                alphaPairsChanged += innerL(i,oS)
                print("fullSet, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged))
            iter += 1
        else:
            nonBoundIs = np.nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]
            for i in nonBoundIs:
                alphaPairsChanged += innerL(i,oS)
                print("non-bound, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged))
            iter += 1
        if entireSet:
            entireSet = False
        elif (alphaPairsChanged == 0):
            entireSet = True
        print("iteration number: %d" % iter)
    return oS.b,oS.alphas
# 畫圖
def showClassifer(dataMat, labelMat, alphas,w ,b):
    data_plus = []
    data_minus = []
    #注意六 把原始點畫出來，橙色的點對應的標籤是-1，藍色的點對應的標籤是1
    for i in range(len(dataMat)):
        if labelMat[i] > 0:
            data_plus.append(dataMat[i])
        else:
            data_minus.append(dataMat[i])

    data_plus_np = np.array(data_plus)
    data_minus_np = np.array(data_minus)
    plt.scatter(np.transpose(data_plus_np)[0],np.transpose(data_plus_np)[1],s = 30, alpha=0.7)
    plt.scatter(np.transpose(data_minus_np)[0], np.transpose(data_minus_np)[1], s=30, alpha=0.7)
    #注意七:畫出訓練出來的超平面
    x1 = max(dataMat)[0]
    x2 = min(dataMat)[0]
    a1, a2 = w
    b = float(b)
    a1 = float(a1[0])
    a2 = float(a2[0])
    y1, y2 = (-b - a1 * x1) / a2,(-b - a1 * x2) / a2 # 縱座標max  min
    plt.plot([x1,x2],[y1,y2])
    # 注意八 畫出向量機點，和那些“忽略”的點
    # 帶有紅色圓圈的是支持向量機點即間隔線上的點，帶有黑色的點是間隔線內的點
    for i, alpha in enumerate(alphas): #將一個可遍歷的數據對象組合爲一個索引序列，同時列出數據和數據下標
        if 0.6 > abs(alpha) > 0:
            x, y = dataMat[i]
            plt.scatter([x],[y],s=150,c='none',alpha=0.7,linewidths=1.5,edgecolors='red')
        if 0.6 == abs(alpha):
            x, y = dataMat[i]
            plt.scatter([x],[y],s=150,c='none',alpha=0.7,linewidths=1.5,edgecolors='black')
    plt.show()

Mytest

# -*- coding: utf-8 -*-#
import smoP as svm
x=[[1,8],[3,20],[1,15],[3,35],[5,35],[4,40],[7,80],[6,49],[1.5,25],[3.5,45],[4.5,50],[6.5,15],[5.5,20],[5.8,74],[2.5,5]]
y=[1,1,-1,-1,1,-1,-1,1,-1,-1,-1,1,1,-1,1]
b,alphas = svm.smoP(x,y,0.6,0.001,40)
w = svm.calcWs(x,y,alphas)
svm.showClassifer(x,y,alphas, w, b)

太難了，扣腳
一直有個疑惑，爲啥每次迭代有時候會自己找到幾個“算法合理”的最優，一定要迭代到40纔是最優，所有每次還要重新計算。

核函數

對鬆弛變量的理解:
1、取值存在一個最優解，當鬆弛變量太大，支持向量機太少，也就是說我們利用了很少的點去決策，顯然結果不好，正如上面體現的那樣，測試集的錯誤在上升，當鬆弛變量太小，支持向量機太多，也就是我們基本利用了所有樣本點。
2、最重要的還是要通過調試找到RBF最佳的超參數值。（引用https://blog.csdn.net/weixin_42001089/article/details/83420109）

# -*- coding: utf-8 -*-
import random
import numpy as np
import matplotlib.pyplot as plt

#加載訓練集
def loadDataSet(filename):
    dataMat = []
    labelMat = []
    fr = open(filename)
    for line in fr.readlines():
        lineArr = line.strip().split('\t')
        #s.strip(rm) 刪除s字符串中開頭、結尾處，位於rm刪除序列的字符;split按括號內的進行返回一個列表
        dataMat.append([float(lineArr[0]),float(lineArr[1])])
        labelMat.append(float(lineArr[2]))
    return dataMat,labelMat
# 定義了一個類作爲數據結構來存儲一些信息;建立一個類存放基本數據以及alphas的緩存
class optStruct:
    def __init__(self,dataMatIn, classLabels, C, toler, kTup):
        self.X = dataMatIn             #數據
        self.labelMat = classLabels    #標籤
        self.C = C                     #容忍度
        self.tol = toler               #誤差容忍度
        self.m = np.shape(dataMatIn)[0]#數據個數，給出行；1的話是給出列
        self.alphas = np.mat(np.zeros((self.m,1))) #alpha值 (m,1)
        self.b = 0                     #常數項
        self.eCache = np.mat(np.zeros((self.m,2)))#保存誤差和下表 (m,2)
        self.K = np.mat(np.zeros((self.m,self.m)))
        for i in range(self.m):
            self.K[:,i] = kernelTrans(self.X, self.X[i,:], kTup)
#輔助函數一: 創建一個樣本點返回來的隨機整數（0，m）,即選擇alpha（i，j）
def selectJrand(i,m):
    j = i
    while (j == i):
        j  = int(random.uniform(0,m)) #返回一個浮點數 N，取值範圍爲0,m
    return j
# 輔助函數二：將aj規劃到[0,C]範圍
def ClipAlpha(aj,H,L):
    if aj > H:
        aj = H
    if aj < L:
        aj = L
    return aj
# 徑向高斯核函數（RBF）
def kernelTrans(X, A, kTup):
    m,n = np.shape(X)
    K = np.mat(np.zeros((m,1)))
    if kTup[0]=='lin':
        K = X * A.T
    elif kTup[0]=='rbf':
        for j in range(m):
            deltaRow = X[j,:] - A
            K[j] = deltaRow*deltaRow.T
        K = np.exp(K/(-1*kTup[1]**2))
    else:
        raise NameError('Houston We Have a Problem -- That Kernel is not recognized')
    return K
# 建立計算錯誤率的函數
def calcEk(oS, k):
    fxk = float(np.multiply(oS.alphas, oS.labelMat).T *\
          (oS.X*oS.X[k,:].T) +oS.b)
    Ek = fxk - float(oS.labelMat[k])
    return Ek

# 啓發式找j:選擇最長步長的那個（就是選擇|Ei-Ej|最大）
def selectJ(i, oS, Ei):
    maxK = -1
    maxDeltaE = 0
    Ej = 0
    oS.eCache[i] = [1,Ei] #誤差緩存。1st 列爲1時表示有效（計算好了誤差）
    validEcachelist = np.nonzero(oS.eCache[:,0].A)[0] #1\A.將matrix類型轉換成array類型2\返回數組中不爲0的元素下標
    if(len(validEcachelist )) > 1:
        for k in validEcachelist:
            if k == i:
                continue
            Ek = calcEk(oS,k)
            deltaE = abs(Ei - Ek)
            if (deltaE > maxDeltaE):
                maxK = k
                maxDeltaE = deltaE
                Ej = Ek
        return maxK, Ej
    else:
        j = selectJrand(i, oS.m)
        Ej = calcEk(oS,j)
    return j, Ej
#更新錯誤率Ek
def updateEk(oS,k):
    Ek = calcEk(oS,k)
    oS.eCache[k] = [1,Ek]
#更新錯誤率Ek
def updateEk(oS,k):
    Ek = calcEk(oS,k)
    oS.eCache[k] = [1,Ek]
# 更新b並且返回兩個alpha參數是否改變的情況
def innerL(i,oS):
    Ei = calcEk(oS, i)
    if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or\
    ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):#不滿足KKT
        j,Ej = selectJ(i, oS, Ei)  # 尋找|Ei-Ej|最大的j
        alphaIold = oS.alphas[i].copy()
        alphaJold = oS.alphas[j].copy()
        if (oS.labelMat[i] != oS.labelMat[j]):
            L = max(0, oS.alphas[j] - oS.alphas[i])
            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])
        else:
            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)
            H = min(oS.C, oS.alphas[j] + oS.alphas[i])
        if L==H:
            print("L==H")
            return 0
        eta = 2.0 * oS.K[i,j]-oS.K[i,i]-oS.K[j,j]
        if eta >= 0:
            print("eta>=0")
            return 0
        oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/eta
        oS.alphas[j] = ClipAlpha(oS.alphas[j],H,L)
        updateEk(oS, j)
        if (abs(oS.alphas[j] - alphaJold) < oS.tol):
            print("j not moving enough")
            return 0
        oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])
        updateEk(oS, i)
        b1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,i] - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[i,j]
        b2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,j]- oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[j,j]

        if (0 < oS.alphas[i] < oS.C):
            oS.b = b1
        elif (0 < oS.alphas[j] < oS.C):
            oS.b = b2
        else:
            oS.b = (b1 + b2)/2.0
        return 1
    else:
        return 0
# smpK
def smoK(dataMatIn, classLabels, C, toler, maxIter,kTup=('lin', 0)):
    oS = optStruct(np.mat(dataMatIn),np.mat(classLabels).transpose(),C,toler, kTup)
    iter = 0
    entireSet = True
    alphaPairsChanged = 0
    while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):
        alphaPairsChanged = 0
        if entireSet:
            for i in range(oS.m):
                alphaPairsChanged += innerL(i,oS)
                print("fullSet, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged))
            iter += 1
        else:
            nonBoundIs = np.nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]
            for i in nonBoundIs:
                alphaPairsChanged += innerL(i,oS)
                print("non-bound, iter: %d i:%d, pairs changed %d" % (iter,i,alphaPairsChanged))
            iter += 1
        if entireSet:
            entireSet = False
        elif (alphaPairsChanged == 0):
            entireSet = True
        print("iteration number: %d" % iter)
    return oS.b,oS.alphas
#作用是使用訓練集去訓練SVM模型，然後分別統計該模型在訓練集上和測試集上的錯誤率。
def testRbf(data_train,data_test):
    dataArr,labelArr = loadDataSet(data_train)
    b,alphas = smoK(dataArr,labelArr,200,0.0001,10000,('rbf',5))
    datMat = np.mat(dataArr)
    labelMat = np.mat(labelArr).transpose()
    svInd = np.nonzero(alphas)[0] #通過\alpha構建權重w時是只用到是支持向量機那些點即\alpha > 0那些點
    sVs = datMat[svInd]
    labelSV = labelMat[svInd]
    print("there are %d Support Vectors" % np.shape(sVs)[0])
    m,n = np.shape(datMat)
    errorCount = 0
    for i in range(m):
        kernelEval = kernelTrans(sVs,datMat[i,:],('rbf',1.3))
        predict = kernelEval.T * np.multiply(labelSV,alphas[svInd]) + b
        if np.sign(predict) != np.sign(labelArr[i]):
            errorCount +=1
    print("the training error rate is: %f" %(float(errorCount)/m))
    dataArr_test, labelArr_test = loadDataSet(data_test)
    errorCount_test = 0
    datMat_test = np.mat(dataArr_test)
    labelMat = np.mat(labelArr_test).transpose()
    m,n = np.shape(datMat_test)
    for i in range(m):
        kernelEval = kernelTrans(sVs, datMat_test[i,:],('rbf',0.1))
        predict = kernelEval.T * np.multiply(labelSV,alphas[svInd]) + b
        if np.sign(predict) != np.sign(labelArr[i]): #f=sign(x):x>0:1;x=0:0;x<0:-1
            errorCount +=1
    print("the testing error rate is: %f" %(float(errorCount_test)/m))
    return dataArr,labelArr,alphas

# 畫圖
def showClassifer(dataMat, labelMat, alphas):
    data_plus = []
    data_minus = []
    #注意六 把原始點畫出來，橙色的點對應的標籤是-1，藍色的點對應的標籤是1
    for i in range(len(dataMat)):
        if labelMat[i] > 0:
            data_plus.append(dataMat[i])
        else:
            data_minus.append(dataMat[i])

    data_plus_np = np.array(data_plus)
    data_minus_np = np.array(data_minus)
    plt.scatter(np.transpose(data_plus_np)[0],np.transpose(data_plus_np)[1],s = 30, alpha=0.7)
    plt.scatter(np.transpose(data_minus_np)[0], np.transpose(data_minus_np)[1], s=30, alpha=0.7)
    # 帶有紅色圓圈的是支持向量機點即間隔線上的點
    for i, alpha in enumerate(alphas): #將一個可遍歷的數據對象組合爲一個索引序列，同時列出數據和數據下標
        if  abs(alpha) > 0:
            x, y = dataMat[i]
            plt.scatter([x],[y],s=150,c='none',alpha=0.7,linewidths=1.5,edgecolors='red')
    plt.show()

# -*- coding: utf-8 -*-
import smoK as svm
traindata = 'C:\\Users\\Lenovo\\Desktop\\Train_data.txt'
testdata = 'C:\\Users\\Lenovo\\Desktop\\Test_data.txt'
TraindataArr, TrainlabelArr, alphas = svm.testRbf(traindata,testdata)
svm.showClassifer(TraindataArr,TrainlabelArr,alphas)