AdaBoost算法的單機實現

AI課上的作業,我用python實現一下

實現的目標如下:

首先簡單介紹一下Adaboost算法,也叫做自增強適應。用來將多個弱的分類器/預測器變成一個比較強的分類器。具體算法如下:

[0]弱分類器我只實現了簡單的二分類,然後用AdaBoost算法將這些分類器訓練成一個強分類器。

[1]強分類器由多個弱分類器加權之後結合而成,每一個弱分類器則有自己的判別規則。這裏簡單起見只用了最最簡單的水平|垂直分類器。

[2]弱分類器是一個閉包函數,綁定了相關參數,沒有使用類。

[3]強分類器由多個弱分類器組成,也是一個閉包函數

[4]中間很多語句是用來輸出參數的,所以看着可能稍微有一點亂。

[5]關於弱分類器的選擇,將數據點離散化之後就構造出許多弱分類器,利用誤差公式來選擇一個最好的。

[6]將數據分成了train和test…因爲數據實在太少了,我也懶得搞了。。。

import matplotlib.pyplot as plt
import math

blue = [(-2,19),(13,15),(-8,11),(-14,14),(6,19),(-12,4),(15,13)]#,(26,13),(24,9)]
red  = [(4,-3),(5,-2),(1,6),(-6,5),(5,3),(11,-5),(-7,9)]#,(6,-3)]#,(15,-1),(16,-8)]
test1 = [((26,13),-1),((24,9),-1),((15,-1),1),((16,-8),1),((6,-3),1)]
points = [(point,1)for point in red]
points.extend((point,-1) for point in blue)
e = math.e

def create_weak_classifiers(points):
    def create_weak_classifier(_p,_theta,_kind):
        #val is input,theta is threshold,p determines area
        def weak_classifier(point):
            val = point[0] if _kind == 1 else point[1]
            if _p*val<_p*_theta:
                return 1
            else:
                return -1
        return weak_classifier

    #remove repeated position and sort
    x_index = list(set(sorted(p[0][0]  for p in points)))
    y_index = list(set(sorted(p[0][1]  for p in points)))
    n = len(x_index)
    h = {}
    attributes = {}
    ind = 0

    for pos,t in zip((x_index,y_index),(1,2)):
        for i in range(len(pos)-1):
            for p in (-1,1):
                m = (pos[i]+pos[i+1])/2
                h[ind] = create_weak_classifier(p,m,t)
                attributes[ind] = (p,m,t)
                ind +=1
    return h,attributes
def create_strong_classifier(H):
    def strong_calssifier(point):
        ret = 0
        for alpha,ind,weak_cls in H:
            ret += alpha*weak_cls(point)
        return 1 if ret>0 else -1#sign(ret)
    return strong_calssifier
def compute_error(points,f,D):
    error = 0
    for (i,(point,kind)) in enumerate(points):
        error += D[i]*(kind!=f(point))

    return error
def select_best_weak(points,h,D):
    min_error = 10**3
    ind,f = None,None
    for index,weak_cls in h.items():
        error = compute_error(points,weak_cls,D)
        if error < min_error:
            min_error = error
            ind,best_cls = index,weak_cls
    return ind,best_cls,min_error

#initialize parameter
N = len(blue)+len(red)
D = [1/N]*N 
T = 10
alpha = [0]*T
h,attributes = create_weak_classifiers(points)
H = []
if __name__ == '__main__':
    for t in range(T):
        print('Iteration{} :'.format(t),end='  ')
        ind,current_weak,error = select_best_weak(points,h,D)
        assert(error<0.5)
        alpha[t] = 0.5*math.log((1-error)/error)
        H.append((alpha[t],ind,current_weak))
        print('alpha :{}'.format(alpha[t]),end='\t')
        #update D
        Z = sum(D[i]*e**(-alpha[t]*points[i][1]*current_weak(points[i][0])) for i in range(N))
        for i in range(N):
            point,kind = points[i]
            frac_1 = D[i]*e**(-alpha[t]*kind*current_weak(point))
            D[i] = frac_1/Z

        #print h(x)
        attr = attributes[ind]
        p,m,kind = attr[0],attr[1],attr[2]
        print('h(x):{} * {} < {}'.format(p,'x' if kind==1 else 'y',m),end='\t')
        #compute the CE
        strong_cls = create_strong_classifier(H)
        total_error = compute_error(points,strong_cls,D)
        print('CE:{0:.3f}'.format(total_error))
        if total_error==0:
            count = 0
            for point,kind in test1:
                count += (kind!=strong_cls(point))
            print('test data'count)
            break