scrapy复习（豆瓣250电影存mysql）

前言

最近在家太无聊，开始看以前在慕课学的课程。顺便学习下scrapy这个以前半懂不懂的框架。Python最火爬虫框架Scrapy入门与实践

代码

首先新建项目

在cmd中输入scrapy startobject xxx

第二创建爬虫

进入spider文件夹下，scrapy genspider 自己爬虫名字 域名

第三确定目标

编写item.py确定要保存的字段

# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class DoubanItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #序号，电影名称，电影介绍，星级，电影评论数，电影描述
    serial_number = scrapy.Field()
    movie_name = scrapy.Field()
    introduce = scrapy.Field()
    star = scrapy.Field()
    evaluate = scrapy.Field()
    depict = scrapy.Field()

第四解析网页获取自己要保存的字段

# -*- coding: utf-8 -*-
import scrapy
from items import DoubanItem

class DoubanSpiderSpider(scrapy.Spider):
    #爬虫名字
    name = 'douban_spider'
    #允许域名
    allowed_domains = ['movie.douban.com']
    #入库url
    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        movie_list = response.xpath('//div[@class="article"]//ol[@class="grid_view"]/li')
        for i in movie_list:

            douban_item = DoubanItem()
            douban_item['serial_number'] = i.xpath(".//div[@class ='item']//em/text()")\
                .extract_first()
            douban_item['movie_name'] = i.xpath(
                ".//div[@class='info']//div[@class='hd']/a/span[@class = 'title'][1]/text()")\
                .extract_first()
            content = i.xpath(".//div[@class='info']//div[@class='bd']/p[1]/text()").extract()
            for j in content:
                content_i = "".join(j.split())
                douban_item['introduce'] = content_i
            douban_item['star'] = i.xpath(".//span[@class='rating_num']/text()").extract_first()
            douban_item['evaluate'] = i.xpath(".//div[@class = 'star']//span[4]/text()").extract_first()
            douban_item['depict'] = i.xpath(".//p[@class='quote']/span/text()").extract_first()
            # douban_item['describe'] = douban_item['describe'].strip()
            yield douban_item

        next_link = response.xpath("//span[@class = 'next']/link/@href").extract()
        if next_link:
            next_link = next_link[0]
            yield scrapy.Request('https://movie.douban.com/top250'+next_link,callback=self.parse)

第五编写mysql链接

编写pipline.py

class MySQLPipeline(object):
    def __init__(self):
        self.connect = connect(
            host='0.0.0.0',
            port=3306,
            db='scrapy',
            user='root',
            passwd='xxxxx',
            charset='utf8',
            use_unicode=True)
        # 连接数据库
        self.cursor = self.connect.cursor()
        # 使用cursor()方法获取操作游标

    def process_item(self, item, spider):
        self.cursor.execute(
            """INSERT INTO douban (serial_number, movie_name, introduce, star, evaluate, depict) VALUES (%s, %s, %s, %s, %s, %s)""",
            (item['serial_number'],
             item['movie_name'],
             item['introduce'],
             item['star'],
             item['evaluate'],
             item['depict']
             ))
        # 执行sql语句，item里面定义的字段和表字段一一对应
        self.connect.commit()
        # 提交
        return item
    # 返回item

    def close_spider(self, spider):
        self.cursor.close()
        # 关闭游标
        self.connect.close()
        # 关闭数据库连接

注意！字段不能为describe、desc等，这是mysql保留关键字。这个坑查了很久！再次提醒自己复习mysql。。。。。

第六更改middleware.py隐藏自己

import random
# user agent 列表
USER_AGENT_LIST = [
    'MSIE (MSIE 6.0; X11; Linux; i686) Opera 7.23',
    'Opera/9.20 (Macintosh; Intel Mac OS X; U; en)',
    'Opera/9.0 (Macintosh; PPC Mac OS X; U; en)',
    'iTunes/9.0.3 (Macintosh; U; Intel Mac OS X 10_6_2; en-ca)',
    'Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)',
    'iTunes/4.2 (Macintosh; U; PPC Mac OS X 10.2)',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:5.0) Gecko/20100101 Firefox/5.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:9.0) Gecko/20100101 Firefox/9.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20120813 Firefox/16.0',
    'Mozilla/4.77 [en] (X11; I; IRIX;64 6.5 IP30)',
    'Mozilla/4.8 [en] (X11; U; SunOS; 5.7 sun4u)'
]
# 随机生成user agent
USER_AGENT = random.choice(USER_AGENT_LIST)

借鉴python爬虫之scrapy中user agent浅谈（两种方法）

第七更改setting.py

55行开启middleware，67行开启pipline
DOWNLOAD_DELAY为设置延迟更改0…5
USER_AGENT修改（如果使有middleware则可以不用）

后续进行neo4j知识图谱学习。