轉載自： CDA數據分析師
1、數據獲取
https://y.qq.com/n/yqq/album/0009C3rp3Kfwg0.html#comment_box
評論區的內容是被封裝在json中
複製此條json數據，放到在線json解析中
分析數據結構
comment_list = json_data['comment']['commentlist']
# 暱稱
nick_name = [i.get('nick') for i in comment_list]
# 評論內容
content = [i.get('rootcommentcontent') for i in comment_list]
# 評論時間
comment_time = [i.get('time') for i in comment_list]
# 點贊數
praise_num = [i.get('praisenum') for i in comment_list]
整個代碼
# 導入包
import pandas as pd
import time
import requests
import json
from fake_useragent import UserAgent
def get_qq_comment(page_num):
# 存儲數據
df_all = pd.DataFrame()
for i in range(page_num):
# 打印進度
print('我正在獲取第{}頁的信息'.format(i))
# 獲取URL
url = 'https://c.y.qq.com/base/fcgi-bin/fcg_global_comment_h5.fcg?g_tk_new_20200303=5381&g_tk=5381&loginUin=0&hostUin=0&format=json&inCharset=utf8&outCharset=GB2312¬ice=0&platform=yqq.json&needNewCode=0&cid=205360772&reqtype=2&biztype=2&topid=12924001&cmd=8&needmusiccrit=0&pagenum={}&pagesize=25'.format(i)
# 添加headers
headers = {
'user-agent': UserAgent().random
}
# 發起請求
try:
r = requests.get(url, headers=headers)
except Exception as e:
print(e)
continue
# 解析網頁
json_data = json.loads(r.text)
# 獲取數據
comment_list = json_data['comment']['commentlist']
# 暱稱
nick_name = [i.get('nick') for i in comment_list]
# 評論內容
content = [i.get('rootcommentcontent') for i in comment_list]
# 評論時間
comment_time = [i.get('time') for i in comment_list]
# 點贊數
praise_num = [i.get('praisenum') for i in comment_list]
# 存儲數據
df = pd.DataFrame({
'nick_name': nick_name,
'content': content,
'comment_time': comment_time,
'praise_num': praise_num
})
# 追加數據
df_all = df_all.append(df, ignore_index=True)
# 休眠一秒
time.sleep(1)
return df_all
# 運行函數
df = get_qq_comment(page_num=5)
df.to_excel('C:\\Users\\Administrator\\Desktop\\mojito.xlsx',index = False)
只要5頁內容，試一下就行了
2、數據處理
讀入數據
df = pd.read_excel('C:\\Users\\Administrator\\Desktop\\mojito.xlsx')
查看重複值和空值
print(df.duplicated().sum()) # 0
print(df.isnull().sum())
'''
nick_name 0
content 0
comment_time 0
praise_num 0
dtype: int64
'''
df.info
時間格式轉換
df['comment_time']
# 原格式
'''
0 1592708649
1 1592708347
2 1592708274
3 1592708154
4 1592708011
120 1592636465
121 1592636412
122 1592636120
123 1592636047
124 1592636017
Name: comment_time, Length: 125, dtype: int64
'''
def transform_time(time_second):
time_array = time.localtime(time_second)
otherStyleTime = time.strftime('%Y-%m-%d %H:%M:%S', time_array)
return otherStyleTime
# 時間數據處理
df['comment_time'] = df['comment_time'].apply(lambda x: transform_time(x))
'''
0 2020-06-21 11:04:09
1 2020-06-21 10:59:07
2 2020-06-21 10:57:54
3 2020-06-21 10:55:54
4 2020-06-21 10:53:31
120 2020-06-20 15:01:05
121 2020-06-20 15:00:12
122 2020-06-20 14:55:20
123 2020-06-20 14:54:07
124 2020-06-20 14:53:37
Name: comment_time, Length: 125, dtype: object
'''
content 評論內容初步處理（正則—表達式）
import re
pattern = re.compile(r'\[em\](.*?)\[/em\]')
df['content'] = df.content.str.replace(pattern, '')
df.head()
按時間排序
comment_num = df.comment_time.str.split(':').str[0].value_counts().sort_index()
'''
索引
年 - 月- 日 小時 這個時點評論人數
2020-06-20 14 3
2020-06-20 15 15
2020-06-20 16 6
2020-06-20 17 15
2020-06-20 18 11
2020-06-20 19 12
2020-06-20 20 4
2020-06-20 21 10
2020-06-20 22 5
2020-06-20 23 10
2020-06-21 00 9
2020-06-21 01 1
2020-06-21 02 3
2020-06-21 05 1
2020-06-21 06 1
2020-06-21 07 1
2020-06-21 08 3
2020-06-21 09 7
2020-06-21 10 7
2020-06-21 11 1
Name: comment_time, dtype: int64
'''
產生時間序列數據
# 去掉年份
x_line1 = [i.replace('2020-','') for i in comment_num.index.to_list()]
'''
['06-20 14',
'06-20 15',
'06-20 16',
'06-20 17',
'06-20 18',
...
'''
# 這個時點評論人數
y_line1 = comment_num.values.tolist()
# [3, 15, 6, 15, 11, 12, 4, 10, 5, 10, 9, 1, 3, 1, 1, 1, 3, 7, 7, 1]
3、數據可視化
導入所需包
import jieba
import stylecloud
from pyecharts.charts import Pie, Bar, Map, Line, WordCloud, Page
from pyecharts import options as opts
from pyecharts.globals import SymbolType
Mojito評論人數走勢圖
c = (
Line() #初始化
.add_xaxis(x_line1) # X軸
# Y軸
.add_yaxis('', # 系列名稱
y_line1,# 系列數據
# 標記點配置項
markpoint_opts=opts.MarkPointOpts(data=[
opts.MarkPointItem(type_='max', name='最大值'),# 標記最大值
opts.MarkPointItem(type_='min', name='最小值') # 標記最小值
]))
# 全局配置項
.set_global_opts(
# 標題設置
title_opts=opts.TitleOpts('Mojito評論人數走勢圖'),
# 軸標籤設置
xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate='30')), # 標籤旋轉
)
# 系統配置項
.set_series_opts(
# 標籤配置項
label_opts=opts.LabelOpts(is_show=False), # 不顯示標籤
# 線配置項
linestyle_opts=opts.LineStyleOpts(width=3)) # 線寬3
.render("line_base.html")
)
QQ音樂評論詞雲圖
def get_cut_words(content_series):
# 讀入停用詞表
stop_words = []
with open(r"C:\\Users\\Administrator\\Desktop\\stop_words.txt", 'r') as f:
lines = f.readlines()
for line in lines:
stop_words.append(line.strip())
# 添加關鍵詞
my_words = ['周杰倫', '一首歌', '好好聽', '方文山', '30多歲']
for i in my_words:
jieba.add_word(i)
# 自定義停用詞
my_stop_words = ['歌有', '真的', '這首', '一首', '一點',
'反正', '一段', '一句', '首歌', '啊啊啊',
'哈哈哈', '轉發', '微博', '那段', '他會'
]
stop_words.extend(my_stop_words)
# 分詞
content=';'.join([ str(c) for c in df['content'].tolist()])
word_num = jieba.lcut(content)
# 條件篩選
word_num_selected = [i for i in word_num if i not in stop_words and len(i)>=2]
return word_num_selected
text1 = get_cut_words(content_series=df.content)
stylecloud.gen_stylecloud(text=' '.join(text1),
max_words=1000,
collocations=False,
font_path='C:\\Windows\\Fonts\\simhei.ttf',
icon_name='fas fa-music',
size=624,
gradient='vertical' ,
palette='cartocolors.diverging.TealRose_2',
output_name='C:\\Users\\Administrator\\Desktop\\QQ音樂評論詞雲圖.png')
