在一個難數據集上應用 AdaBoost
此較難數據即邏輯迴歸算法中從疝氣病症預測病馬的數據
說明:
將 horseColicTraining2.txt 和 horseColicTest2.txt 放在當前目錄下。
from numpy import *
單層決策樹生成的函數
"""
單層決策樹分類函數
Parameters:
dataMatrix - 數據矩陣
dimen - 第dimen列，也就是第幾個特徵
threshVal - 閾值
threshIneq - 標誌
Returns:
retArray - 分類結果
第一個函數stumpClassify()是通過閾值比較對數據進行分類的。
所有在閾值一邊的數據會分到類別1，而在另外一邊的數據分到類別+1。該函數可以通過數組過
濾來實現，首先將返回數組的全部元素設置爲1，然後將所有不滿足不等式要求的元素設置爲1。
可以基於數據集中的任一元素進行比較，同時也可以將不等號在大於、小於之間切換
"""
def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):
#初始化retArray爲1
retArray = ones((shape(dataMatrix)[0],1))
if threshIneq == 'lt':
#如果小於閾值,則賦值爲-1
retArray[dataMatrix[:,dimen] <= threshVal] = -1.0
else:
#如果大於閾值,則賦值爲-1
retArray[dataMatrix[:,dimen] > threshVal] = -1.0
return retArray
"""
找到數據集上最佳的單層決策樹
Parameters:
dataArr - 數據矩陣
classLabels - 數據標籤
D - 樣本權重
Returns:
bestStump - 最佳單層決策樹信息
minError - 最小誤差
bestClasEst - 最佳的分類結果
"""
def buildStump(dataArr,classLabels,D):
dataMatrix = mat(dataArr)
labelMat = mat(classLabels).T
m,n = shape(dataMatrix)
numSteps = 10.0; bestStump = {}; bestClasEst = mat(zeros((m,1)))
#最小誤差初始化爲正無窮大
minError = inf
#遍歷所有特徵
for i in range(n):
#找到特徵中最小的值和最大值
rangeMin = dataMatrix[:,i].min(); rangeMax = dataMatrix[:,i].max();
#計算步長
stepSize = (rangeMax-rangeMin)/numSteps
for j in range(-1,int(numSteps)+1):
#大於和小於的情況，均遍歷。lt:less than，gt:greater than
for inequal in ['lt', 'gt']: #go over less than and greater than
#計算閾值
threshVal = (rangeMin + float(j) * stepSize)
#計算分類結果
predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal)
#初始化誤差矩陣
errArr = mat(ones((m,1)))
#分類正確的,賦值爲0
errArr[predictedVals == labelMat] = 0
#計算誤差
weightedError = D.T*errArr #calc total error multiplied by D
print("split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f" % (i, threshVal, inequal, weightedError))
#找到誤差最小的分類方式
if weightedError < minError:
minError = weightedError
bestClasEst = predictedVals.copy()
bestStump['dim'] = i
bestStump['thresh'] = threshVal
bestStump['ineq'] = inequal
return bestStump,minError,bestClasEst
基於單層決策樹的 AdaBoost 訓練過程
"""
使用AdaBoost算法提升弱分類器性能
Parameters:
dataArr - 數據矩陣
classLabels - 數據標籤
numIt - 最大迭代次數
Returns:
weakClassArr - 訓練好的分類器
aggClassEst - 類別估計累計值
"""
def adaBoostTrainDS(dataArr,classLabels,numIt=40):
#收集不同的單層決策樹樁的信息
weakClassArr = []
m = shape(dataArr)[0]
#初始化權重
D = mat(ones((m,1))/m) #init D to all equal
aggClassEst = mat(zeros((m,1)))
for i in range(numIt):
#構建單層決策樹
bestStump,error,classEst = buildStump(dataArr,classLabels,D)
print("D:",D.T)
#計算弱分類器權重alpha,使error不等於0,因爲分母不能爲0
alpha = float(0.5*log((1.0-error)/max(error,1e-16)))
#存儲弱學習算法權重
bestStump['alpha'] = alpha
#存儲單層決策樹
weakClassArr.append(bestStump)
print("classEst: ",classEst.T)
#計算e的指數項
expon = multiply(-1*alpha*mat(classLabels).T,classEst)
D = multiply(D,exp(expon))
#根據樣本權重公式，更新樣本權重
D = D/D.sum()
#更新累計類別估計值------這裏包括目前已經訓練好的每一個分類器
aggClassEst += alpha*classEst
print("aggClassEst: ",aggClassEst.T)
#計算誤差,這裏注意布爾值的邏輯運算
aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T,ones((m,1)))
errorRate = aggErrors.sum()/m
print("total error: ",errorRate)
#誤差爲0，退出循環
if errorRate == 0.0: break
return weakClassArr
自適應數據加載函數
def loadDataSet(fileName): #general function to parse tab -delimited floats
numFeat = len(open(fileName).readline().split('\t')) #get number of fields
dataMat = []; labelMat = []
fr = open(fileName)
for line in fr.readlines():
lineArr =[]
curLine = line.strip().split('\t')
for i in range(numFeat-1):
lineArr.append(float(curLine[i]))
dataMat.append(lineArr)
labelMat.append(float(curLine[-1]))
return dataMat,labelMat
datArr, labelArr = loadDataSet('horseColicTraining2.txt')
使用 AdaBoost 函數進行分類
classifierArr = adaBoostTrainDS(datArr, labelArr, 50)
在測試集上進行測試
"""
AdaBoost分類函數
Parameters:
datToClass - 待分類樣例
classifierArr - 訓練好的分類器
Returns:
分類結果
"""
def adaClassify(datToClass,classifierArr):
dataMatrix = mat(datToClass)#do stuff similar to last aggClassEst in adaBoostTrainDS
m = shape(dataMatrix)[0]
aggClassEst = mat(zeros((m,1)))
#遍歷所有分類器，進行分類
for i in range(len(classifierArr)):
classEst = stumpClassify(dataMatrix, classifierArr[i]['dim'],\
classifierArr[i]['thresh'],\
classifierArr[i]['ineq'])#call stump classify
aggClassEst += classifierArr[i]['alpha']*classEst
print(aggClassEst)
return sign(aggClassEst)
datArr, labelArr = loadDataSet('horseColicTest2.txt')
labelPred = adaClassify(datArr,classifierArr)
errorRate = mean(labelPred != mat(labelArr).T)
print("total error: ",errorRate)
繪製 ROC 曲線
def plotROC(predStrengths, classLabels):
import matplotlib.pyplot as plt
cur = (1.0,1.0) #cursor
ySum = 0.0 #variable to calculate AUC
numPosClas = sum(array(classLabels)==1.0)
yStep = 1/float(numPosClas); xStep = 1/float(len(classLabels)-numPosClas)
sortedIndicies = predStrengths.argsort()#get sorted index, it's reverse
fig = plt.figure()
fig.clf()
ax = plt.subplot(111)
#loop through all the values, drawing a line segment at each point
for index in sortedIndicies.tolist()[0]:
if classLabels[index] == 1.0:
delX = 0; delY = yStep;
else:
delX = xStep; delY = 0;
ySum += cur[1]
#draw line from cur to (cur[0]-delX,cur[1]-delY)
ax.plot([cur[0],cur[0]-delX],[cur[1],cur[1]-delY], c='b')
cur = (cur[0]-delX,cur[1]-delY)
ax.plot([0,1],[0,1],'b--')
plt.xlabel('False positive rate'); plt.ylabel('True positive rate')
plt.title('ROC curve for AdaBoost horse colic detection system')
ax.axis([0,1,0,1])
plt.show()
print("the Area Under the Curve is: ",ySum*xStep)
修改 adaBoostTrainDS 函數最後一行，返回預測的概率值 aggClassEst：
def adaBoostTrainDS(dataArr,classLabels,numIt=40):
weakClassArr = []
m = shape(dataArr)[0]
D = mat(ones((m,1))/m) #init D to all equal
aggClassEst = mat(zeros((m,1)))
for i in range(numIt):
bestStump,error,classEst = buildStump(dataArr,classLabels,D)#build Stump
print("D:",D.T)
alpha = float(0.5*log((1.0-error)/max(error,1e-16)))#calc alpha, throw in max(error,eps) to account for error=0
bestStump['alpha'] = alpha
weakClassArr.append(bestStump) #store Stump Params in Array
print("classEst: ",classEst.T)
expon = multiply(-1*alpha*mat(classLabels).T,classEst) #exponent for D calc, getting messy
D = multiply(D,exp(expon)) #Calc New D for next iteration
D = D/D.sum()
#calc training error of all classifiers, if this is 0 quit for loop early (use break)
aggClassEst += alpha*classEst
print("aggClassEst: ",aggClassEst.T)
aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T,ones((m,1)))
errorRate = aggErrors.sum()/m
print("total error: ",errorRate)
if errorRate == 0.0: break
return weakClassArr, aggClassEst
對訓練樣本測試畫ROC：
datArr, labelArr = loadDataSet('horseColicTraining2.txt')
classifierArr, aggClassEst = adaBoostTrainDS(datArr, labelArr, 10)
參考文檔：
1.https://blog.csdn.net/red_stone1