支持向量機SVM (SMO) 的可視化 demo

做了一個demo可以動態顯示支持向量機的更新狀態, 其中藍色的線是skleran算出來的結果, 紅色的線是自己算出來的, 一般迭代 5-10次就和sklearn的 (藍色的線) 高度重合了, 正確率不到100% 是數據本身的問題,沒法用線性劃分到100%正確

比較了網上幾個版本, 一直達不到理想的效果, 後來發現是b一直大幅跳躍 , 便做了一點修改, 首先alpha2是遍歷所有的點, 遇到需要更新且可以更新的點就選擇這個點, 主要修改 alpha1 的選擇標準, alpha1的一定優先選擇可能是支持向量的 0< alpha1< C , 因爲b的更新邏輯是如果其中一個點是sv,就以它爲基準更新, 所以如果alpha1 和alpha2 都不是sv, b的變動就會很大, 這樣會達不到收斂的效果, 實際測試修改後收斂的非常快

    def selectAlpha1_index(self, alpha2_index):
        # 非零alpha的是sv的機率大
        E2 = self.EiCatch[alpha2_index]
        nonZeroList=[]
        for i in range(self.rows):
            alpha=self.alpha[i,0]
            if(0<alpha<self.C):
                nonZeroList.append(i)

        if (len(nonZeroList) == 0):
            return self.selectJrand(alpha2_index)
        else:
            maxDiff = 0
            j = -1
            for i in range(len(nonZeroList)):
                row = nonZeroList[i]
                if (row == alpha2_index):
                    continue
                else:
                    E1 = self.EiCatch[row]
                    if (abs(E1 - E2) > maxDiff):
                        maxDiff = abs(E1 - E2)
                        j = row
            if (j == -1):
                return self.selectJrand(alpha2_index)
            else:
                return j

1到5次的迭代效果:

最後附上全套代碼

import numpy as np
from numpy import *
import matplotlib.pyplot as plt
import time
from sklearn import svm

def getDataArray(list, xMat):
    x = ndarray(len(list))
    y = ndarray(len(list))
    count = 0
    for i in list:
        x[count] = xMat[i, 0]
        y[count] = xMat[i, 1]
        count += 1
    return x, y


def loadData():
    fp = open('testSet_a.txt')
    linesData = fp.readlines()
    dataSet = []
    labelSet = []
    for oneLine in linesData:
        oneLine = oneLine.split('*')
        dataSet.append([float(oneLine[0].strip()), float(oneLine[1].strip())])
        labelSet.append(float(oneLine[2].strip()))
    return dataSet, labelSet


class SVM:
    def __init__(self, xSet, yArray, C=None, floatingPointError=0.0001):
        self.xMat = mat(xSet)  # (48,2)
        self.yMat = mat(yArray).T  # (48,1)
        self.rows = self.xMat.shape[0]
        self.cols = self.xMat.shape[1]
        self.alpha = mat(np.zeros(self.rows)).T  # (48,1)
        self.w = None  # 最後返回,計算過程不需要
        self.b = 0
        self.C = C  # C=None時表示hard margin
        self.fpe = floatingPointError

        self.trainCount = 0  # 記錄訓練次數
        self.K = np.matmul(self.xMat, self.xMat.transpose())
        # Ei 緩存
        self.EiCatch = np.zeros(self.rows)
        self.updateEi_catch()

    def predict(self, xArray):
        resultList = []
        for i in range(len(xArray)):
            # v1=np.multiply(xArray[i], self.w)
            v = np.sum(np.multiply(xArray[i], self.w)) + self.b
            if (v > 0):
                resultList.append(1)
            else:
                resultList.append(-1)
        return resultList

    def score(self, xArray, yArray):
        resultList = self.predict(xArray)
        count = 0
        for i in range(len(yArray)):
            if (resultList[i] == yArray[i]):
                count += 1
        return round(count / len(yArray) * 100, 2)

    def train(self, maxCount,debug):
        self.trainCount = 0
        while (self.trainCount < maxCount):
            self.update_allPoints(debug)
            self.trainCount += 1
        # 打印alpha信息
        print(self.alpha)

        return self.w, self.b

    def update_allPoints(self,debug=None):
        count=0
        for alpha2_index in range(self.rows):
            if (self.check_alpha2_needUpdate(alpha2_index)):
                alpha1_index = self.selectAlpha1_index(alpha2_index)
                self.update_alpha_and_b(alpha1_index, alpha2_index)


                # 計算w
                self.w = np.matmul(np.multiply(self.yMat, self.alpha).T, self.xMat)
                if(debug):
                    # 打印alpha信息
                    print(self.alpha)
                    #畫圖
                    self.classifyDataAndPlot()
                    print("調整次數:{}".format(count+1))
                    count+=1
                    # 打印ei信息
                    print(self.EiCatch)

    def check_alpha2_needUpdate(self, alpha2_index):
        Ei = self.EiCatch[alpha2_index]
        yi = self.yMat[alpha2_index, 0]
        alpha2 = self.alpha[alpha2_index, 0]
        fx=self.cal_Fx(alpha2_index)

        if(alpha2<0 or alpha2>self.C):
            return True

        if(yi==1 and fx>=1):
            return False
        elif(yi==-1 and fx<=-1):
            return False

        #再來看看是否有足夠的空間調整
        # Ei不爲零的,alpha應該是0如果不是就要調整,alpha2調整量就是 -yi*Ei,如果是正的, alpha增加,但如果已經是C的話就不用處理了

        alpha2_change_direction = -yi * Ei
        if (alpha2_change_direction > self.fpe and alpha2 < self.C):
            return True
        elif (alpha2_change_direction < -self.fpe and alpha2 > 0):
            return True
        else:
            return False

    def update_alpha_and_b(self, alpha1_index, alpha2_index):
        alpha1_old = self.alpha[alpha1_index, 0]
        alpha2_old = self.alpha[alpha2_index, 0]
        y1 = self.yMat[alpha1_index, 0]
        y2 = self.yMat[alpha2_index, 0]

        alpha2_new_chiped = self.get_alpha2_new_chiped(alpha1_index, alpha2_index)
        alpha1_new = alpha1_old + y1 * y2 * (alpha2_old - alpha2_new_chiped)
        b_new = self.get_b_new(alpha1_index, alpha2_index, alpha1_new, alpha2_new_chiped)
        # 最後更新數據
        alpha2_new_chiped = round(alpha2_new_chiped, 5)
        alpha1_new = round(alpha1_new, 5)
        b_new = round(b_new, 5)

        self.alpha[alpha1_index, 0], self.alpha[alpha2_index, 0] = alpha1_new, alpha2_new_chiped
        self.b = b_new
        # 更新EiCatch
        self.updateEi_catch()
        return True

    def get_b_new(self, alpha1_index, alpha2_index, alpha1_new, alpha2_new_chiped):
        alpha1_old = self.alpha[alpha1_index, 0]
        alpha2_old = self.alpha[alpha2_index, 0]
        y1 = self.yMat[alpha1_index, 0]
        y2 = self.yMat[alpha2_index, 0]
        K11 = self.K[alpha1_index, alpha1_index]
        K12 = self.K[alpha1_index, alpha2_index]
        K22 = self.K[alpha2_index, alpha2_index]
        E1 = self.EiCatch[alpha1_index]
        E2 = self.EiCatch[alpha2_index]
        b1New = self.b - E1 + y1 * K11 * (alpha1_old - alpha1_new) + y2 * K12 * (alpha2_old - alpha2_new_chiped)
        b2New = self.b - E2 + y1 * K12 * (alpha1_old - alpha1_new) + y2 * K22 * (alpha2_old - alpha2_new_chiped)
        # 只有符合的alpha_new用來調整b
        if (self.C == None):
            alpha1_valid = True if 0 < alpha1_new < self.fpe else False
            alpha2_valid = True if 0 < alpha2_new_chiped else False
        else:
            alpha1_valid = True if 0 < alpha1_new < self.C else False
            alpha2_valid = True if 0 < alpha2_new_chiped < self.C else False
        if alpha1_valid:
            b = b1New
        elif alpha2_valid:
            b = b2New
        else:
            b = (b1New + b2New) / 2
        return b

    def check_kkt_status(self):
        # yi和alpha的乘積和爲0
        if not (-self.fpe < np.sum(np.multiply(self.yMat, self.alpha)) < self.fpe):
            return False
        # 然後檢查每個alpha
        for i in range(len(self.alpha)):
            if (self.check_satisfiy_kkt_onePoint(i) == False):
                return False
        return True

    def cal_Ei(self, index):
        v = self.cal_Fx(index) - self.yMat[index, 0]
        return round(v,5)

    def cal_Fx(self, index):
        # (1,48) * (48,1)=1
        v = float(np.multiply(self.alpha, self.yMat).T * self.K[:, index] + self.b)
        return round(v, 5)

    def updateEi_catch(self):
        # alpha變動的時候更新
        for i in range(self.rows):
            v=self.cal_Ei(i)
            self.EiCatch[i] = v
        return True

    def check_alpha2_vaild(self, alpha1_index, alpha2_index, Ei_list):
        # 計算更新量是否足夠
        if (alpha1_index == alpha2_index):
            return False
        alpha2_new_chiped = self.get_alpha2_new_chiped(alpha1_index, alpha2_index, Ei_list)
        alpha2_old = self.alpha[alpha2_index, 0]
        if (None == alpha2_new_chiped):
            return False
        else:
            if (abs(alpha2_new_chiped - alpha2_old) > self.fpe):
                return True
            else:
                return False

    def get_alpha2_new_chiped(self, alpha1_index, alpha2_index):
        alpha2_old = self.alpha[alpha2_index, 0]
        y2 = self.yMat[alpha2_index, 0]
        E1 = self.EiCatch[alpha1_index]
        E2 = self.EiCatch[alpha2_index]
        eta = self.K[alpha1_index, alpha1_index] + self.K[alpha2_index, alpha2_index] - 2.0 * self.K[
            alpha1_index, alpha2_index]
        if (eta == 0):
            return None
        try:
            alpha2_new_unc = alpha2_old + (y2 * (E1 - E2) / eta)
            alpha2_new_chiped = self.get_alpha2_chiped(alpha2_new_unc, alpha1_index, alpha2_index)
        except:
            print()

        return alpha2_new_chiped

    def get_alpha2_chiped(self, alpha2_new_unc, alpha1_index, alpha2_index):
        y1 = self.yMat[alpha1_index, 0]
        y2 = self.yMat[alpha2_index, 0]
        alpha1 = self.alpha[alpha1_index, 0]
        alpha2 = self.alpha[alpha2_index, 0]

        if (self.C == None):
            # hard margin
            if (y1 == y2):
                H = alpha1 + alpha2
                L = 0
            else:
                H = None
                L = max(0, alpha2 - alpha1)
        else:
            # soft margin
            if (y1 == y2):
                H = min(self.C, alpha1 + alpha2)
                L = max(0, alpha1 + alpha2 - self.C)
            else:
                H = min(self.C, self.C - alpha1 + alpha2)
                L = max(0, alpha2 - alpha1)

        alpha2_new_chiped = None
        if (alpha2_new_unc < L):
            alpha2_new_chiped = L
        else:
            if H is None:
                alpha2_new_chiped = alpha2_new_unc
            else:
                if (alpha2_new_unc > H):
                    alpha2_new_chiped = H
                else:
                    alpha2_new_chiped = alpha2_new_unc
        return alpha2_new_chiped

    def classifyDataAndPlot(self):
        #把支持向量取出來
        sv_array = []
        for i in range(self.rows):
            if(0<self.alpha[i]<self.C):
                sv_array.append(i)
        print("共有支持向量數量:",len(sv_array))

        #把點區分爲四種,正負例並區分是否是支持向量
        sv_positive_list = []
        sv_negtive_list = []
        no_sv_negtive_list = []
        no_sv_positive_list = []

        for i in range(self.rows):
            yi = self.yMat[i, 0]
            if (i in sv_array):
                if (yi == 1):
                    sv_positive_list.append(i)
                else:
                    sv_negtive_list.append(i)
            else:
                if (yi == 1):
                    no_sv_positive_list.append(i)
                else:
                    no_sv_negtive_list.append(i)

        # 畫點
        sv_p_x, sv_p_y = getDataArray(sv_positive_list, self.xMat)
        sv_n_x, sv_n_y = getDataArray(sv_negtive_list, self.xMat)
        nosv_p_x, nosv_p_y = getDataArray(no_sv_positive_list, self.xMat)
        nosv_n_x, nosv_n_y = getDataArray(no_sv_negtive_list, self.xMat)

        plt.scatter(sv_p_x, sv_p_y, s=20, marker="+", c="r")
        plt.scatter(sv_n_x, sv_n_y, s=20, marker="*", c="blue")
        plt.scatter(nosv_p_x, nosv_p_y, s=20, marker="+", c="orange")
        plt.scatter(nosv_n_x, nosv_n_y, s=20, marker="*", c="g")

        # 畫線
        # w0=self.w[0,0].flatten
        print("w:", self.w)
        print("b:", self.b)

        # 畫 wx+b=0的實線
        X1 = np.linspace(-2, 3, 2).reshape(2, 1)
        X2_0 = (0 - self.b - self.w[0, 0] * X1) / self.w[0, 1]
        plt.plot(X1, X2_0, color='red', linewidth=0.5, linestyle="-")

        # 畫wx+b=+1和wx+b=-1的虛線
        X2_positive = (1-self.b - self.w[0, 0] * X1) / self.w[0, 1]
        X2_negtive = (-1-self.b - self.w[0, 0] * X1) / self.w[0, 1]
        plt.plot(X1, X2_positive, color='red', linewidth=0.5, linestyle="--")
        plt.plot(X1, X2_negtive, color='red', linewidth=0.5, linestyle="--")

        # sklearn,數據是跑sklearn出來的 用這個方法 runWithSkleran
        sk = [-0.93105886, 0.82281036]
        skLearnW = np.array(sk)
        skLearnB = -5.39363898
        X2_sklearn = (0 - skLearnB - skLearnW[0] * X1) / skLearnW[1]
        plt.plot(X1, X2_sklearn, color='blue', linewidth=0.5, linestyle="-")

        # # 在我的 notebook 裏，要設置下面兩行才能顯示中文
        plt.rcParams['font.family'] = ['sans-serif']
        # 如果是在 PyCharm 裏，只要下面一行，上面的一行可以刪除
        plt.rcParams['font.sans-serif'] = ['SimHei']
        plt.rcParams['axes.unicode_minus'] = False  # 解決保存圖像是負號'-'顯示爲方塊的問題
        title="迭代次數:"+str(self.trainCount)+",支持向量數量:"+str(len(sv_array))
        plt.title(title)

        plt.show()

    def selectJrand(self, i):
        j = i
        while (i == j):
            j = int(np.random.uniform(0, self.rows))
        return j

    def selectAlpha1_index(self, alpha2_index):
        # 非零alpha的是sv的機率大
        E2 = self.EiCatch[alpha2_index]
        nonZeroList=[]
        for i in range(self.rows):
            alpha=self.alpha[i,0]
            if(0<alpha<self.C):
                nonZeroList.append(i)

        if (len(nonZeroList) == 0):
            return self.selectJrand(alpha2_index)
        else:
            maxDiff = 0
            j = -1
            for i in range(len(nonZeroList)):
                row = nonZeroList[i]
                if (row == alpha2_index):
                    continue
                else:
                    E1 = self.EiCatch[row]
                    if (abs(E1 - E2) > maxDiff):
                        maxDiff = abs(E1 - E2)
                        j = row
            if (j == -1):
                return self.selectJrand(alpha2_index)
            else:
                return j


def runWithSkleran(trainX, trainY):
    classifier = svm.SVC(kernel='linear')
    # classifier = svm.SVC(C=1.0, kernel='poly')
    # classifier = svm.SVC(C=1.0, kernel='rbf')

    classifier.fit(trainX, trainY)
    value_predict = classifier.predict(trainX)
    count = 0
    errIndex = []
    for i in range(len(value_predict)):
        predict = value_predict[i]
        if (predict == trainY[i]):
            count += 1
        else:
            errIndex.append(i)

    print("準確率{:.2%}".format(count / len(trainY)))
    print("err:", errIndex)
    print('Coefficients:%s, intercept %s' % (classifier.coef_, classifier.intercept_))
    print('Score: %.2f' % classifier.score(trainX, trainY))

    return classifier.coef_[0], classifier.intercept_[0]


def runMySvm():
    xSet, ySet = loadData()
    classifier = SVM(xSet, ySet, C=2)

    # debug模式每次迭代更新一次圖,可以看動畫的效果
    w, b = classifier.train(100,debug=False)
    score = classifier.score(xSet, ySet)
    print("正確率:", score)
    classifier.classifyDataAndPlot()

def runSklearn():
    trainX, trainY = loadData()
    w, b = runWithSkleran(trainX, trainY)
    print("w",w)
    print("b",b)


if __name__ == '__main__':
    import sys
    #跑sklearn
    # runSklearn()

    runMySvm()
    sys.exit()