本次作業的要求來自於:https://edu.cnblogs.com/campus/gzcc/GZCC-16SE1/homework/2822
初始化jieba環境:
1. 下載一長篇中文小說。
2. 從文件讀取待分析文本。
3. 安裝並使用jieba進行中文分詞。
pip install jieba
import jieba
jieba.lcut(text)
4. 更新詞庫,加入所分析對象的專業詞彙。
jieba.add_word('天罡北斗陣') #逐個添加
jieba.load_userdict(word_dict) #詞庫文本文件
test=open(r"..\python1\threeCountry.txt", "r",encoding="utf-8").read() File=open(r"..\python1\stops_chinese.txt", "r",encoding="utf-8") jieba.load_userdict(r"..\python1\比較全的三國人名.txt") #停詞表 stops = File.read().split('\n') ch="《》\n:,。、-!?"
轉換代碼:scel_to_text
# -*- coding: utf-8 -*- import struct import os # 拼音表偏移, startPy = 0x1540; # 漢語詞組表偏移 startChinese = 0x2628; # 全局拼音表 GPy_Table = {} # 解析結果 # 元組(詞頻,拼音,中文詞組)的列表 # 原始字節碼轉爲字符串 def byte2str(data): pos = 0 str = '' while pos < len(data): c = chr(struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0]) if c != chr(0): str += c pos += 2 return str # 獲取拼音表 def getPyTable(data): data = data[4:] pos = 0 while pos < len(data): index = struct.unpack('H', bytes([data[pos],data[pos + 1]]))[0] pos += 2 lenPy = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] pos += 2 py = byte2str(data[pos:pos + lenPy]) GPy_Table[index] = py pos += lenPy # 獲取一個詞組的拼音 def getWordPy(data): pos = 0 ret = '' while pos < len(data): index = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] ret += GPy_Table[index] pos += 2 return ret # 讀取中文表 def getChinese(data): GTable = [] pos = 0 while pos < len(data): # 同音詞數量 same = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] # 拼音索引表長度 pos += 2 py_table_len = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] # 拼音索引表 pos += 2 py = getWordPy(data[pos: pos + py_table_len]) # 中文詞組 pos += py_table_len for i in range(same): # 中文詞組長度 c_len = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] # 中文詞組 pos += 2 word = byte2str(data[pos: pos + c_len]) # 擴展數據長度 pos += c_len ext_len = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] # 詞頻 pos += 2 count = struct.unpack('H', bytes([data[pos], data[pos + 1]]))[0] # 保存 GTable.append((count, py, word)) # 到下個詞的偏移位置 pos += ext_len return GTable def scel2txt(file_name): print('-' * 60) with open(file_name, 'rb') as f: data = f.read() print("詞庫名:", byte2str(data[0x130:0x338])) # .encode('GB18030') print("詞庫類型:", byte2str(data[0x338:0x540])) print("描述信息:", byte2str(data[0x540:0xd40])) print("詞庫示例:", byte2str(data[0xd40:startPy])) getPyTable(data[startPy:startChinese]) getChinese(data[startChinese:]) return getChinese(data[startChinese:]) if __name__ == '__main__': # scel所在文件夾路徑 in_path = r"F:\text" #修改爲你的詞庫文件存放文件夾 # 輸出詞典所在文件夾路徑 out_path = r"F:\text" # 轉換之後文件存放文件夾 fin = [fname for fname in os.listdir(in_path) if fname[-5:] == ".scel"] for f in fin: try: for word in scel2txt(os.path.join(in_path, f)): file_path=(os.path.join(out_path, str(f).split('.')[0] + '.txt')) # 保存結果 with open(file_path,'a+',encoding='utf-8')as file: file.write(word[2] + '\n') os.remove(os.path.join(in_path, f)) except Exception as e: print(e) pass
5. 生成詞頻統計
for w in tokens: if len(w)==1: continue else: wordict[w] = wordict.get(w,0)+1
7. 排除語法型詞彙,代詞、冠詞、連詞等停用詞。
stops
tokens=[token for token in wordsls if token not in stops]
wordlist=jieba.lcut(test) tokens=[token for token in wordlist if token not in stops]
8. 輸出詞頻最大TOP20,把結果存放到文件裏。
for i in range(20): print(wordsort[i])
9. 生成詞雲。
pd.DataFrame(wordsort).to_csv('three.csv', encoding='utf-8') txt = open('three.csv','r',encoding='utf-8').read() cut_text=''.join(txt) from wordcloud import WordCloud mywc=WordCloud().generate(cut_text) import matplotlib.pyplot as plt plt.imshow(mywc) plt.axis("off") plt.show() mywc.to_file(r"F:\txt\threestory.png")
整體代碼:
# _*_ coding: utf-8 _*_
import jieba
import pandas as pd
test=open(r"..\python1\threeCountry.txt", "r",encoding="utf-8").read()
File=open(r"..\python1\stops_chinese.txt", "r",encoding="utf-8")
jieba.load_userdict(r"..\python1\比較全的三國人名.txt")
#停詞表
stops = File.read().split('\n')
ch="《》\n:,。、-!?"
for c in ch:
test = test.replace(c,'')
#更新詞庫,加入所分析對象的專業詞彙
jieba.add_word('哎喲不錯喲')
#中文切詞
wordlist=jieba.lcut(test)
tokens=[token for token in wordlist if token not in stops]
wordict={}
for w in tokens:
if len(w)==1:
continue
else:
wordict[w] = wordict.get(w,0)+1
wordsort=list(wordict.items())
wordsort.sort(key= lambda x:x[1],reverse=True)
#輸出詞頻最大TOP20
for i in range(20):
print(wordsort[i])
pd.DataFrame(wordsort).to_csv('three.csv', encoding='utf-8')
txt = open('three.csv','r',encoding='utf-8').read()
cut_text=''.join(txt)
from wordcloud import WordCloud
mywc=WordCloud().generate(cut_text)
import matplotlib.pyplot as plt
plt.imshow(mywc)
plt.axis("off")
plt.show()
mywc.to_file(r"F:\txt\threestory.png")