寫在開頭的話:在學習《機器學習實戰》的過程中發現書中很多代碼並沒有註釋,這對新入門的同學是一個挑戰,特此貼出我對代碼做出的註釋,僅供參考,歡迎指正。
1、進行文本分類
def loadDataSet():
postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
['stop', 'posting', 'stupid', 'worthless', 'garbage'],
['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
classVec = [0, 1, 0, 1, 0, 1] #ddf
return postingList, classVec
#功能:創建不重複詞的列表
#輸入:數據集
#輸出:不重複詞的列表
def createVocabList(dataSet):
vocabSet = set([])#創建空集合
for document in dataSet:
vocabSet = vocabSet | set(document)#或操作表示創建並集
return list(vocabSet)
#功能:輸入文檔中的單詞在詞彙表中是否出現
#輸入:詞彙表,輸入文檔
#輸出:文檔向量
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0] * len(vocabList)#創建和詞彙表等長的向量
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:
print "the word: %s is not in my Vocabulary!" % word
return returnVec
#功能:樸素貝葉斯分類器訓練函數
#輸入:文檔向量,標籤向量
#輸出:任意文檔屬於侮辱性文檔的概率,詞彙在侮辱性文檔中出現概率,詞彙在非侮辱性文檔中出現概率
def trainNB0(trainMatrix, trainCategory):
numTrainDocs = len(trainMatrix)#trainMatrix行數
numWords = len(trainMatrix[0])#trainMatrix列數
pAbusive = sum(trainCategory) / float(numTrainDocs)#任意文檔屬於侮辱性文檔的概率
p0Num = ones(numWords)#拉普拉斯修正
p1Num = ones(numWords)#拉普拉斯修正
p0Denom = 2.0#拉普拉斯修正,只有兩個標籤
p1Denom = 2.0#拉普拉斯修正,只有兩個標籤
for i in range(numTrainDocs):
if trainCategory[i] == 1:#屬於侮辱性文檔
p1Num += trainMatrix[1]#侮辱性文檔數量
p1Denom += sum(trainMatrix[i])
else:#不屬於侮辱性文檔
p0Num += trainMatrix[i]#非侮辱性文檔數量
p0Denom += sum(trainMatrix[i])
p1Vect = log(p1Num / p1Denom)#防止下溢出,自然對數處理
p0Vect = log(p0Num / p0Denom)#防止下溢出,自然對數處理
return p0Vect, p1Vect, pAbusive
#功能:判斷屬於哪一類
#輸入:要判斷向量,詞彙在侮辱性文檔中出現概率,詞彙在非侮辱性文檔中出現概率,任意文檔屬於侮辱性文檔的概率
#輸出:判斷值
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
p1 = sum(vec2Classify * p1Vec) + log(pClass1)
p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
if p1 > p0:
return 1
else:
return 0
#功能:輸入文檔中的單詞在詞彙表中是否出現
#輸入:詞彙表,輸入文檔
#輸出:文檔向量
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0] * len(vocabList) # 創建和詞彙表等長的向量
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:
print "the word: %s is not in my Vocabulary!" % word
return returnVec
def bagOfWords2VecMN(vocabList, inputSet):
returnVec = [0]*len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] += 1
return returnVec
#功能:便利函數
#輸入:無
#輸出:無
def testingNB():
label = ['ham', 'spam']#將結果以文本形式呈現
listOPosts, listClasses = loadDataSet()
myVocabList = createVocabList(listOPosts)
trainMat = []
for postinDoc in listOPosts:
trainMat.append(setOfWords2Vec(myVocabList, postinDoc))
p0V, p1V, pAb = trainNB0(array(trainMat), array(listClasses))
testEntry = ['love', 'my', 'dalmation']
thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
print testEntry, 'classified as: ', label[classifyNB(thisDoc, p0V, p1V, pAb)]
testEntry = ['stupid', 'garbage']
thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
print testEntry, 'classified as: ', label[classifyNB(thisDoc, p0V, p1V, pAb)]
2、過濾垃圾郵件
#功能:切分文本
#輸入:需要切分文本
#輸出:切分好的文本
def textParse(bigString):
import re
listOfTokens = re.split(r'\W*', bigString)#\W表示非單詞字符,*表示匹配前一個字符0次或無限次
#參考鏈接:http://www.cnblogs.com/huxi/archive/2010/07/04/1771073.html
return [tok.lower() for tok in listOfTokens if len(tok) > 2]#列表推導式
#功能:測試算法
#輸入:無
#輸出:測試正確率
def spamTest():
docList = []#文檔列表
classList = []#標籤列表
fullText = []#單詞列表
for i in range(1, 26):
wordList = textParse(open('email/spam/%d.txt' % i).read())#打開spam文檔,切分文本
docList.append(wordList)#append向列表尾部添加一個新的元素,將整個wordList添加進去,即文本爲基本單位
fullText.extend(wordList)#extend向列表尾部添加wordList的所有元素,即單詞爲基本單位
classList.append(1)#標籤表示爲spam
wordList = textParse(open('email/ham/%d.txt' % i).read()) # 打開ham文檔,切分文本
docList.append(wordList) # append向列表尾部添加一個新的元素,將整個wordList添加進去,即文本爲基本單位
fullText.extend(wordList) # extend向列表尾部添加wordList的所有元素,即單詞爲基本單位
classList.append(0)#標籤表示爲ham
vocabList = createVocabList(docList)#創建不重複詞的列表
trainingSet = range(50)
testSet = []
for i in range(10):#在trainingSet中隨機刪除十個數,刪除的作爲測試集,沒有刪除的作爲訓練集
randIndex = int(random.uniform(0, len(trainingSet)))
testSet.append(trainingSet[randIndex])
del(trainingSet[randIndex])
trainMat = []
trainClasses = []
for docIndex in trainingSet:#得測試集和測試標籤
trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
trainClasses.append(classList[docIndex])
p0V, p1V, pSpam = trainNB0(array(trainMat), array(trainClasses))
errorCount = 0
for docIndex in testSet:
wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
if classifyNB(array(wordVector), p0V, p1V, pSpam) != classList[docIndex]:#判斷測試是否正確
errorCount += 1
print "classifiction error", docList[docIndex]
print 'the error rate is: ', float(errorCount) / len(testSet)
3、RSS應用
#功能:得出現頻率最高的30個詞
#輸入:詞彙表,文本
#輸出:頻率最高的30個詞出現的個數和詞
def calcMostFreq(vocabList, fullText):
import operator
freqDict = {}#建立字典
for token in vocabList:
freqDict[token] = fullText.count(token)
sortedFreq = sorted(freqDict.iteritems(), key = operator.itemgetter(1), reverse = True)#按值從小到大的順序排序
return sortedFreq[:30]#返回前30 單詞出現個數和詞
#功能:rss功能
#輸入:
#輸出:
def localWords(feed1, feed0):
import feedparser
docList = []
classList = []
fullText = []
minLen = min(len(feed1['entries']), len(feed0['entries']))
for i in range(minLen):
wordList = textParse(feed1['entries'][i]['summary'])
docList.append(wordList)
fullText.extend(wordList)
classList.append(1)
wordList = textParse(feed0['entries'][i]['summary'])
docList.append(wordList)
fullText.extend(wordList)
classList.append(0)
vocabList = createVocabList(docList)
top30Words = calcMostFreq(vocabList, fullText)
for pairW in top30Words:
if pairW[0] in vocabList:
vocabList.remove(pairW[0])
trainingSet = range(2 * minLen)
testSet = []
for i in range(20):
randIndex = int(random.uniform(0, len(trainingSet)))
testSet.append(trainingSet[randIndex])
del(trainingSet[randIndex])
trainMat = []
trainClasses = []
for docIndex in trainingSet:
trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
trainClasses.append(classList[docIndex])
p0V, p1V, pSpam = trainNB0(array(trainMat), array(trainClasses))
errorCount = 0
for docIndex in testSet:
wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
if classifyNB(array(wordVector), p0V, p1V, pSpam) != classList[docIndex]:
errorCount += 1
print 'the error rate is: ', float(errorCount) / len(testSet)
return vocabList, p0V, p1V
def getTopWords(nf, sf):
import operator
vocabList, p0V, p1V = localWords(nf, sf)
topNY = []
topSF = []
for i in range(len(p0V)):
if p0V[i] > -6.0 : topSF.append((vocabList[i], p0V[i]))
if p1V[i] > -6.0 : topNY.append((vocabList[i], p1V[i]))
sortedSF = sorted(topSF, key = lambda pair : pair[1], reverse = True)
print "SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**"
for item in sortedSF:
print item[0]
sortedNY = sorted(topNY, key=lambda pair: pair[1], reverse=True)
print "NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**"
for item in sortedNY:
print item[0]