Python_文本分析_困惑度計算

這篇博文介紹一個困惑度的神仙方法，困惑度是用來確定最佳主題數一種方式。

• 本代碼使用4折交叉驗證
• LDA裏面有兩個參數需要根據自己的數據等進行設定

import csv
import datetime
import re
import pandas as pd
import numpy as np
import jieba
import matplotlib.pyplot as plt
import jieba.posseg as jp, jieba
import gensim
from snownlp import seg
from snownlp import SnowNLP
from snownlp import sentiment
from gensim import corpora, models
from gensim.models import CoherenceModel
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
import warnings
warnings.filterwarnings("ignore")

# Load data
comment = pd.read_csv(r"good.csv", header = 0, index_col = False, engine='python',encoding = 'utf-8')
csv_data = comment[[(len(str(x)) > 100) for x in comment['segment']]]
print(csv_data.shape)

# 設置四折交叉驗證
kf = KFold(n_splits = 4, shuffle=False) 

perplexity_train_all = []
perplexity_test_all = []
    
for train_index , test_index in kf.split(list(csv_data['segment'].values)):  
    train = [ csv_data.iloc[x,7] for x in train_index]
    test = [ csv_data.iloc[x,7] for x in test_index]
    
    vectoriser = CountVectorizer(stop_words = 'english', max_features=1000)
    doc_train = vectoriser.fit_transform(train)
    features = vectoriser.get_feature_names()
    doc_test = vectoriser.fit_transform(test)

    perplexity_train = []
    perplexity_test = []
    
    # 這兩個參數可以根據自己的需要設定
    alpha = 0.005
    beta = 0.1

    # 進行迭代的topic可以適當增加，這裏只用10以內進行計算
    for topics in range(1, 10):
        # Fit LDA to the data 
        LDA = LatentDirichletAllocation(n_components = topics, doc_topic_prior = alpha, topic_word_prior = beta, max_iter=300, learning_method='batch')
        
        # 用來監督處理進程
        news_lda = LDA.fit(doc_train)
        perplexity_train.append(news_lda.perplexity(doc_train))
        perplexity_test.append(news_lda.perplexity(doc_test))
        print(topics, end = '   ')
        
    perplexity_train_all.append(perplexity_train)
    perplexity_test_all.append(perplexity_test)

根據上面代碼計算出的困惑度，可以進行如下作圖，也在這裏展示出來，只作出數據中的兩折交叉驗證的數據及其對應的均值，僅供參考。

# 求均值
perplexity = [(perplexity_train_all[0][i]+perplexity_train_all[1][i])/2 for i in range(9)]

plt.plot(range(1,10),perplexity_train_all[0], label='prep_1', color='r') 
plt.plot(range(1,10),perplexity_train_all[1], label='prep_2', color='b') 
plt.plot(range(1,10),perplexity,label='perp_ave',color='r',marker='o', markerfacecolor='blue',markersize=5) 

plt.xlabel("Num Topics")
plt.ylabel("Perplexity")
plt.legend(loc='best')

plt.savefig(r"C:\Users\lenovo\Desktop\cm.jpeg",dpi = 600)