編寫NLP處理程序的套路學習2-文本相似度度量

1、原理

    文本相似度的度量有很多種方法，特定詞出現頻度，整體文本風格等。本文將使用tf-idf方式，通過cosin相似度度量兩個文本的相似度。     tf爲詞頻代表token frequence

    idf爲你文檔頻率，代表（所有文檔的數目）/包含 該單詞的文檔出現頻率）
    1+log(doc_num/doc_contain_thisWord_num)
    每個單詞的詞頻逆文檔頻率的計算方法爲tf[word] * idf[word]
    將所有文檔中的單詞構成一個詞典，每個單詞用t用一個長度爲len(文檔數)的向量表示，向量中的每一個值具體表示含義如下：如果該單詞出現在文檔中就用tf-idf值替代當前詞，如果該單詞未出現在該文檔中則用0表示。
    文本相似度可以表示爲：
$similarity =\frac{ {\sum {A * B}} }{{ \sqrt{ \sum{A^2}}} * { \sqrt{ \sum{B^2}}}}$

2、代碼

    這段代碼我就不添加太多註釋了，大多可以直接理解的。

import nltk
import math
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

#創建類
class TextSimilarityExample:
    #定義屬性
    def __init__(self):
        self.statments = [
            "ruled india",
            "So many kindom rlued india",
            "Chalukyas ruled inda",
            "your kindom is a good king"
        ]
    #獲取詞頻tokenFrequence字典
    def TF(self, sentence):
        words  = nltk.word_tokenize(sentence.lower())
        freq = nltk.FreqDist(words)
        dictionary = {}
        for key in freq.keys():
            norm = freq[key]/ float(len(words))
            dictionary[key] = norm
        return dictionary

    #獲取逆文檔頻率
    def IDF(self):
        def idf(TotalNumberOfDocuments, NumberOfDocumentWithThisWord):
            return 1.0 - math.log(TotalNumberOfDocuments/NumberOfDocumentWithThisWord)
        numberOfDoc = len(self.statments)
        uniqueWords = {}
        idfValues = {}
        for sentence in self.statments:
            for word in nltk.word_tokenize(sentence.lower()):
                if word not in uniqueWords:
                    uniqueWords[word] = 1
                else:
                    uniqueWords[word] += 1
        for word in uniqueWords:
            idfValues[word] = idf(numberOfDoc, uniqueWords[word])
        return idfValues

    #根據公式得到詞頻逆文檔頻率
    def TF_IDF(self, query):
        words = nltk.word_tokenize(query.lower())
        idf = self.IDF()
        vectors = {}
        for sentence in self.statments:
            tf = self.TF(sentence)
            for word in words:
                tfv = tf[word] if word in tf else 0.0
                idfv = idf[word] if word in idf else 0.0
                mul = tfv * idfv
                if word not in vectors:
                    vectors[word] = []
                vectors[word].append(mul)
        return vectors


    def displayVectors(self, vectors):
        print(self.statments)
        for word in vectors:
            print("{} --> {}".format(word, vectors[word]))

    def cosineSimilarity(self):
        vec = TfidfVectorizer()
        matrix = vec.fit_transform(self.statments)
        for j in range(1,5):
            i = j -1
            print("\t similarity of document {} with others".format(i))
            similarity = cosine_similarity(matrix[i:j], matrix)
            print(similarity)

    def demo(self):
        query = self.statments[0]
        vec = self.TF_IDF(query)
        self.displayVectors(vec)
        self.cosineSimilarity()

if __name__ == "__main__":
    ts = TextSimilarityExample()
    ts.demo()