python爬取豆瓣電影Top250並進行數據分析

源碼：Gitee
歡迎star~
實現爬取數據，存儲到sqlite3，使用flask進行展示，同時，使用wordcloud生成詞雲圖片和使用Echart進行圖表展示

一、requirements

beautifulsoup4==4.9.1
bs4==0.0.1
click==7.1.2
cycler==0.10.0
Flask==1.1.2
itsdangerous==1.1.0
jieba==0.42.1
Jinja2==2.11.2
kiwisolver==1.2.0
MarkupSafe==1.1.1
matplotlib==3.2.1
numpy==1.18.4
Pillow==7.1.2
pyparsing==2.4.7
python-dateutil==2.8.1
six==1.15.0
soupsieve==2.0.1
Werkzeug==1.0.1
wordcloud @ file:python_reptile/flask/static/extend/wordcloud-1.7.0-cp36-cp36m-win32.whl
xlwt==1.3.0

二、獲取並存儲數據

爬取豆瓣TOP250數據，並存儲到數據庫

步驟：

1. 定義爬取地址

2. 獲取URL的數據列表

   通過User-Agent，得到指定一個URL的網頁內容

3. 存儲到sqlite數據庫（數據庫名：movie.db，表名：movie250）

# -*- coding:utf-8 -*-
 
# date： 2020-5-10
# author： jingluo
import sys
from bs4 import BeautifulSoup
import sqlite3
import re
import urllib.request, urllib.error
import xlwt

# 搜索規則
findLink = re.compile(r'<a href="(.*?)">')
findImageSrc = re.compile(r'<img.*src="(.*?)"', re.S) # re.S讓換行符包含在字符中
findTitle = re.compile(r'<span class="title">(.*)</span>')
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
findJudge = re.compile(r'<span>(\d*)人評價</span>')
findInq = re.compile(r'<span class="inq">(.*)</span>')
findBd = re.compile(r'<p class="">(.*?)</p>', re.S)

def main():
	# 1. 定義爬取網址
	base_url = "https://movie.douban.com/top250?start="
	# 2. 獲取數據列表
	data_list = getData(base_url)
	# 3. 定義數據庫名稱
	dbpath = "movie.db"
	# 4. 存儲到sqlite數據庫
	saveData2DB(data_list, dbpath)

# 獲取數據列表
def getData(base_url):
	data_list = []
	for i in range(0, 10):
		url = base_url + str(i*25)
		html = askURl(url)
		
		# 逐一解析網頁
		soup = BeautifulSoup(html, "html.parser")
		for item in soup.find_all("div", class_="item"):
			data = []
			item = str(item)

			link = re.findall(findLink, item)[0]
			data.append(link)
			imgSrc = re.findall(findImageSrc, item)[0]
			data.append(imgSrc)
			titles = re.findall(findTitle, item)
			if len(titles) == 2:
				ctitle = titles[0]
				data.append(ctitle)
				otitle = titles[1].replace("/", "")
				data.append(otitle)
			else:
				data.append(titles[0])
				data.append('')
			rating = re.findall(findRating, item)[0]
			data.append(rating)
			judege = re.findall(findJudge, item)[0]
			data.append(judege)
			inq = re.findall(findInq, item)
			if len(inq) != 0:
				data.append(inq[0].replace("。", ""))
			else:
				data.append('')
			bd = re.findall(findBd, item)[0]
			bd = re.sub('<br(\s+)?/>(\s+)?', " ", bd)
			bd = re.sub('/', " ", bd)
			data.append(bd.strip())
			data_list.append(data)
	return data_list

# 得到指定一個URL的網頁內容
def askURl(url):
	# 用戶驗證信息
	head = {"User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"}
	request = urllib.request.Request(url, headers = head)
	html = ""
	try:
		response = urllib.request.urlopen(request)
		html = response.read().decode("utf-8")
	except urllib.error.URLError as e:
		if hasattr(e, "code"):
			print("請求出錯",e.code)
		if hasattr(e, "reason"):
			print("錯誤原因",e.reason)
	return html

# 保存到sqlite數據庫中
def saveData2DB(data_list, dbpath):
	init_db(dbpath)
	conn = sqlite3.connect(dbpath)
	cur = conn.cursor()

	for data in data_list:
		for index in range(len(data)):
			if index == 4 or index == 5:
				continue
			data[index] = '"' +data[index] + '"'
		sql = '''
		insert into movie250
		(
		info_link,pic_link,cname,ename,score,rated,instroduction,info
		)
		values(%s)'''%",".join(data)
		cur.execute(sql)
		conn.commit()
	cur.close()
	conn.close()

# 初始化數據庫
def init_db(dbpath):
	sql = '''
		create table movie250
		(
		id integer primary key autoincrement,
		info_link text,
		pic_link text,
		cname varchar,
		ename varchar,
		score numeric,
		rated numeric,
		instroduction text,
		info text
		);
	'''
	conn = sqlite3.connect(dbpath)
	cursor = conn.cursor()
	cursor.execute(sql)
	conn.commit()
	conn.close()

三、獲取詞雲

1. 讀取數據庫
2. 使用jieba進行分割
3. 使用word_length.txt存儲詞雲長度
4. 將原始圖轉成數組
5. 使用Image和WordCloud初始化圖片
6. 使用pyplot生成和保存圖片

def makeWordCloud():
	# 準備詞雲所需的詞
	con = sqlite3.connect('movie.db')
	cur = con.cursor()
	sql = 'select instroduction from movie250'
	data = cur.execute(sql)
	text = ""
	for item in data:
		text = text + item[0]
	cur.close()
	con.close()

	cut = jieba.cut(text)
	string = ' '.join(cut)

	filename = 'word_length.txt'
	with open(filename, 'w') as file:
		file.write(str(len(string)))
		file.close()

	img = Image.open(r'../static/assets/img/tree.jpg')
	img_arry = np.array(img) # 將圖片轉換成數組
	wc = WordCloud(
		background_color = 'white',
		mask = img_arry,
		font_path = 'STCAIYUN.TTF' # 字體鎖在位置： C:\Windows\Fonts
		)
	wc.generate_from_text(string)

	# 繪製圖片
	fig = plt.figure(1)
	plt.imshow(wc)
	plt.axis('off') # 是否顯示座標軸
	# plt.show() # 顯示生成的詞雲圖片

	# 輸出詞雲圖片到文件
	plt.savefig(r'../static/assets/img/word.jpg', dpi=800)
	plt.close()

四、完成業務代碼

# -*- coding:utf-8 -*-
 
# date： 2020-5-30
# author： jingluo
from flask import Flask, render_template,request, session
import get_douban_databses
import sqlite3
import os

# 分詞
import jieba
# 繪圖，數據可視化
from matplotlib import pyplot as plt
# 詞雲
from wordcloud import WordCloud
# 圖片處理
from PIL import Image
# 矩陣運算
import numpy as np

# 自定義template路徑
app = Flask(__name__,template_folder="../templates/",
	static_folder='../static/') #應用

# flask的session需要用到的密鑰字符串
app.config["SECRET_KEY"] = "akjsdhkjashdkjhaksk120191101asd"

@app.route("/")
def index():
	try:
		with open('word_length.txt', 'r') as file:
			word_length = file.readline()
			session['word_length'] = word_length
			file.close()
	except:
		word_length = 5633
		session['word_length'] = word_length
	return render_template("template/home.html",word_length = word_length)

@app.route("/home")
def home():
	word_length = session.get('word_length')
	return render_template("template/home.html",word_length = word_length)

@app.route("/movie")
def movie():
	movies = []
	con = sqlite3.connect("movie.db")
	cur = con.cursor()
	sql = "select * from movie250"
	data = cur.execute(sql)
	for item in data:
		movies.append(item)
	cur.close()
	con.close()
	return render_template("template/movie.html",movies = movies)

@app.route("/score")
def score():
	score = []
	number = []
	con = sqlite3.connect("movie.db")
	cur = con.cursor()
	sql = "select score,count(score) from movie250 group by score"
	data = cur.execute(sql)
	for item in data:
		score.append(item[0])
		number.append(item[1])
	cur.close()
	con.close()
	return render_template("template/score.html", score = score, number = number)

# 生成詞雲圖片
def makeWordCloud():
	# 準備詞雲所需的詞
	con = sqlite3.connect('movie.db')
	cur = con.cursor()
	sql = 'select instroduction from movie250'
	data = cur.execute(sql)
	text = ""
	for item in data:
		text = text + item[0]
	cur.close()
	con.close()

	cut = jieba.cut(text)
	string = ' '.join(cut)

	filename = 'word_length.txt'
	with open(filename, 'w') as file:
		file.write(str(len(string)))
		file.close()

	img = Image.open(r'../static/assets/img/tree.jpg')
	img_arry = np.array(img) # 將圖片轉換成數組
	wc = WordCloud(
		background_color = 'white',
		mask = img_arry,
		font_path = 'STCAIYUN.TTF' # 字體鎖在位置： C:\Windows\Fonts
		)
	wc.generate_from_text(string)

	# 繪製圖片
	fig = plt.figure(1)
	plt.imshow(wc)
	plt.axis('off') # 是否顯示座標軸
	# plt.show() # 顯示生成的詞雲圖片

	# 輸出詞雲圖片到文件
	plt.savefig(r'../static/assets/img/word.jpg', dpi=800)
	plt.close()

@app.route("/word")
def word():
	return render_template("template/word.html")

@app.route("/team")
def team():
	return render_template("template/team.html")

if __name__ == '__main__':
	app.config.update(DEBUG=True)
	if not os.path.exists('movie.db'):
		get_douban_databses.main()
	if not os.path.exists('../static/assets/img/word.jpg'):
		makeWordCloud()
	app.run()

五、使用教程

1. git clone https://gitee.com/jingluoonline/python_reptile.git
2. cd python_reptile/flsk/apps
3. 輸入創建虛擬化境的命令virtualenv FlaskPath
4. 進入虛擬環境FlaskPath\Scripts\activate.bat
5. 安裝相關依賴
   1. 其中wordcloud下載有時候會有問題，可以選擇使用whl文件下載，網址https://www.lfd.uci.edu/~gohlke/pythonlibs/#wordcloud找到相應的包下載到本地，進行本地安裝
6. python index.py，有點慢，因爲爬取數據和生成圖片都是在初始化時
7. 瀏覽器輸入http://127.0.0.1:5000/

六、效果圖

1. 主頁
   
2. 電影
   
3. 評分
   
4. 詞雲
   
5. 團隊