- 分析网页成分,歌曲信息都存在图中所示xhr中
- 详细代码实现如下:
import os
import threading
import jieba
import numpy
import html
import requests
import openpyxl
from wordcloud import WordCloud
import PIL.Image as Image
# 设置最大线程锁
thread_lock = threading.BoundedSemaphore(value=10)
# 全局域名地址,用于拼接网址
domain_url = 'https://y.qq.com/n/yqq/song/'
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/80.0.3987.162 Safari/537.36 Edg/80.0.361.109',
'Referer': 'https://y.qq.com/n/yqq/song/0039MnYb0qxYhV.html'
}
# 创建工作簿
wb = openpyxl.Workbook()
# 获取工作簿的活动表
sheet = wb.active
# 工作表重命名
sheet.title = 'song'
# 加表头,分别给A1B1C1单元格赋值
sheet['A1'] = '歌曲名'
sheet['B1'] = '所属专辑'
sheet['C1'] = '播放链接'
# 判断是否有文件夹,无则创建
if not os.path.exists('H:\PythonWorks\QQmusic\评论'):
os.makedirs('H:\PythonWorks\QQmusic\评论')
COMMENT_PATH = 'H:\PythonWorks\QQmusic\评论'
if not os.path.exists("H:\PythonWorks\QQmusic\歌词"):
os.makedirs("H:\PythonWorks\QQmusic\歌词")
LYRIC_PATH = "H:\PythonWorks\QQmusic\歌词"
if not os.path.exists('H:\PythonWorks\QQmusic\热评'):
os.makedirs('H:\PythonWorks\QQmusic\热评')
HOT_COMMENT_PATH = 'H:\PythonWorks\QQmusic\热评'
if not os.path.exists('H:\PythonWorks\QQmusic\词云'):
os.makedirs('H:\PythonWorks\QQmusic\词云')
CIYUN_PATH = 'H:\PythonWorks\QQmusic\词云'
def cut(text):
wordlist_jieba = jieba.cut(text)
space_wordlist = " ".join(wordlist_jieba)
return space_wordlist
# 生成热评词云
def ciyun(path_name, music_name):
with open(path_name, encoding='utf-8') as f:
text = f.read()
text = cut(text)
mask_pic = numpy.array(Image.open('kkx.png'))
wordcloud = WordCloud(
font_path='C:\Windows\Fonts\simfang.ttf',
collocations=False,
max_words=100,
min_font_size=10,
max_font_size=500,
mask=mask_pic
).generate(text)
image = wordcloud.to_image()
dir_path = CIYUN_PATH + '/' + music_name
wordcloud.to_file(dir_path + '.png')
# 下载歌词
def download_lyric(singer, music_name, music_id):
lrc_url = 'https://c.y.qq.com/lyric/fcgi-bin/fcg_query_lyric_yqq.fcg'
params = {
'nobase64': '1',
'musicid': music_id, # 使用上面获取到的id
'-': 'jsonp1',
'g_tk_new_20200303': '5381',
'g_tk': '5381',
'loginUin': '0',
'hostUin': '0',
'format': 'json',
'inCharset': 'utf8',
'outCharset': 'utf-8',
'notice': '0',
'platform': 'yqq.json',
'needNewCode': '0'
}
response = requests.get(lrc_url, params=params, headers=HEADERS)
lrc_json = response.json()
lyric = lrc_json['lyric']
lyric_html = html.unescape(lyric)
f = open(LYRIC_PATH + '/' + singer + '-' + music_name + '.txt', 'a', encoding='utf-8')
f.writelines(lyric_html)
f.close()
print("歌词下载完成...")
# 爬取音乐评论
def parse_comments(singer, music_name, music_id):
comment_url = 'https://c.y.qq.com/base/fcgi-bin/fcg_global_comment_h5.fcg'
n = 1
for x in range(20):
params = {
'g_tk_new_20200303': '5381',
'g_tk': '5381',
'loginUin': '0',
'hostUin': '0',
'format': 'json',
'inCharset': 'utf8',
'outCharset': 'GB2312',
'notice': '0',
'platform': 'yqq.json',
'needNewCode': '0',
'cid': '205360772',
'reqtype': '2',
'biztype': '1',
'topid': music_id,
'cmd': '8',
'needmusiccrit': '0',
'pagenum': x,
'pagesize': '25',
'lasthotcommentid': '',
'domain': 'qq.com',
'ct': '24',
'cv': '10101010'
}
response = requests.get(comment_url, params=params, headers=HEADERS)
# 发起请求
comment_json = response.json()
comments = comment_json['comment']['commentlist']
# 将评论存储到指定的txt
f = open(COMMENT_PATH + '/' + singer + '-' + music_name + '_评论.txt', 'a', encoding='utf-8')
for i in comments:
comment = str(n) + '.' + i['rootcommentcontent'] + '\n------------------------------\n'
f.writelines(comment)
n += 1
f.close()
print("评论下载完成...")
# 爬取音乐热评
def parse_hot_comments(singer, music_name, music_id):
comment_url = 'https://c.y.qq.com/base/fcgi-bin/fcg_global_comment_h5.fcg'
params = {
'g_tk_new_20200303': '5381',
'g_tk': '5381',
'loginUin': '0',
'hostUin': '0',
'format': 'json',
'inCharset': 'utf8',
'outCharset': 'GB2312',
'notice': '0',
'platform': 'yqq.json',
'needNewCode': '0',
'cid': '205360772',
'reqtype': '2',
'biztype': '1',
'topid': music_id,
'cmd': '8',
'needmusiccrit': '0',
'pagenum': 0,
'pagesize': '25',
'lasthotcommentid': '',
'domain': 'qq.com',
'ct': '24',
'cv': '10101010'
}
response = requests.get(comment_url, params=params, headers=HEADERS)
# 发起请求
comment_json = response.json()
comments = comment_json['hot_comment']['commentlist']
path = HOT_COMMENT_PATH + '/' + singer + '-' + music_name + '_热评.txt'
# 将评论存储到指定的txt
f = open(path, 'a', encoding='utf-8')
n = 1
for i in comments:
comment = str(n) + '.' + i['rootcommentcontent'] + '\n------------------------------\n'
f.writelines(comment)
n += 1
f.close()
print("热评下载完成...")
ciyun(path, music_name)
def download(singer, music_name, music_id, album, music_url, sheet):
# 把name、album、link写成列表,用append函数多行写入Excel
sheet.append([music_name, album, music_url])
download_lyric(singer, music_name, music_id)
parse_comments(singer, music_name, music_id)
parse_hot_comments(singer, music_name, music_id)
# 下载完了,解锁
thread_lock.release()
def parse_list(singer, page_number):
# 查询音乐网址
search_url = 'https://c.y.qq.com/soso/fcgi-bin/client_search_cp'
for x in range(page_number):
Params = {
'ct': '24',
'qqmusic_ver': '1298',
'new_json': '1',
'remoteplace': 'txt.yqq.song',
'searchid': '56438082219898629',
't': '0',
'aggr': '1',
'cr': '1',
'catZhida': '1',
'lossless': '0',
'flag_qc': '0',
'p': str(x + 1),
'n': '10',
'w': singer,
'g_tk_new_20200303': '5381',
'g_tk': '5381',
'loginUin': '0',
'hostUin': '0',
'format': 'json',
'inCharset': 'utf8',
'outCharset': 'utf - 8',
'notice': '0',
'platform': 'yqq.json',
'needNewCode': '0'
}
response = requests.get(search_url, params=Params, headers=HEADERS)
music_json = response.json()
# 获取音乐列表
music_list = music_json['data']['song']['list']
for music in music_list:
# song = {'歌曲名': music['name'], '所属专辑': music['album']['name'], '播放链接': domain_url + music['mid'] +
# '.html'}
# 获取音乐对应id
music_id = music['id']
album = music['album']['name']
music_url = domain_url + music['mid'] + '.html'
# 上锁
thread_lock.acquire()
t = threading.Thread(target=download, args=(singer, music['name'], music_id, album, music_url, sheet))
t.start()
# # 下载
# download(singer, music['name'], music_id, album, music_url, sheet)
# 最后保存并命名这个Excel文件
wb.save(singer + '个人单曲排行前' + str(page_number * 10) + '清单.xlsx')
return input('下载完成,是否继续下载(y/n):')
if __name__ == '__main__':
# 设置参数,判断是否需要重复下载
answer = 'y'
while answer != 'n':
# 输入指定参数值进行查询
name = input('请输入要查询的歌手姓名:')
page = int(input('请输入要查询的歌曲页数:'))
answer = parse_list(name, page)
print('下载已全部完成,退出程序。')
- 总结:
通过XHR爬取数据一般要使用json,格式为:
response = requsets.get(url)
json = response.json()
list = json['']['']...