#!/usr/bin/env python
#-*- coding:utf8 -*-
#__author__ = "LiDaguo"
import requests
import re
import xlwt
url = 'https://maoyan.com/board/4?'
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
} # 请求头
def get_page(url):
'''输入网址-返回网页内容'''
try:
response = requests.get(url, headers=headers)
if response.status_code == 200: # 看看是不是正常访问
return response.text
else:
print('获取网页失败')
except Exception as e:
print(e)
def get_info(page):
'''得到网页-提取信息'''
items = re.findall('board-index .*?>(\d+)</i>.*?class="name"><.*?>(.*?)</a></p>.*?<p class="star">.*?' +
'主演:(.*?) .*?</p>.*?<p class="releasetime">(.*?)</p>.*?<p class="score"><i class="integer">' +
'(.*?)</i><i class="fraction">(\d+)</i></p>', page, re.S) # 构造正则表达式
# items是个列表,列表里的每个元素是个元组。每个元组里都包含各个电影的名称、演员、上映时间等信息
for item in items:
data = {}
data['rank'] = item[0]
data['title'] = item[1]
actors = re.sub('\n', '', item[2])
data['actors'] = actors
data['date'] = item[3]
data['score'] = str(item[4]) + str(item[5])
yield data
urls = ['https://maoyan.com/board/4?offset={}'.format(i * 10) for i in range(1)]
DATA = []
for url in urls:
page = get_page(url)
datas = get_info(page) # datas是一个生成器,不是具体数
for item in datas: # 把生成器放到for语句中:每次的item都是生成器返回的值,这两个for循环遥相呼应
DATA.append(item) # 将所有的数据添加到DATA里
f = xlwt.Workbook(encoding='utf-8') # 创建表格
sheet01 = f.add_sheet(u'sheet1', cell_overwrite_ok=True) # 命名sheet
# 表头数据写入excle
sheet01.write(0, 0, 'rank') # 第1行第1列
sheet01.write(0, 1, 'title') # 第1行第2列
sheet01.write(0, 2, 'actors')
sheet01.write(0, 3, 'date')
sheet01.write(0, 4, 'score')
# 写内容
for i in range(len(DATA)):
sheet01.write(i + 1, 0, DATA[i]['rank'])
sheet01.write(i + 1, 1, DATA[i]['title'])
sheet01.write(i + 1, 2, DATA[i]['actors'])
sheet01.write(i + 1, 3, DATA[i]['date'])
sheet01.write(i + 1, 4, DATA[i]['score'])
print('爬取完成', end='')
f.save('E:\\猫眼电影.xls')
效果: