scrapy同步與異步儲存數據

1.同步

class MysqlPipeline(object):
    """
    採用同步的機制寫入mysql
    """

    def __init__(self):
        self.conn = pymysql.connect(host="127.0.0.1", user="root", password="ts123456", db="art_schema", port=3306)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
            insert into jobbole_article(title,url,create_date,fav_nums) 
            values({title},{url},{create_date},{fav_nums});
        """.format(title=item['title'], url=item['url'], create_date=item['create_date'], fav_nums=item['fav_nums'])
        self.cursor.execute(insert_sql)
        self.conn.commit()
        return item

2.異步

class MysqlTwistedPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbparms = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWORD'],
            port=settings['MYSQL_PORT'],
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor,
            use_unicode=True
        )

        dbpool = adbapi.ConnectionPool('pymysql', **dbparms)
        return cls(dbpool)

    def process_item(self, item, spider):
        """
        使用twisted將mysql插入變成異步執行，採用異步的機制寫入mysql
        :param item:
        :param spider:
        :return:
        """
        query = self.dbpool.runInteraction(self.do_insert, item)
        # 處理異常
        query.addErrback(self.handle_error)

    def handle_error(self, failure):
        # 處理異步插入的異常
        print(failure)

    def do_insert(self, cursor, item):
        insert_sql = """
                    insert into jobbole_article(title,url,create_date,fav_nums,url_object_id,front_image_url,
                    front_image_path,praise_nums,comment_nums,tags,content) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);
                """
        cursor.execute(insert_sql,
                       (item['title'], item['url'], item['create_date'], item['fav_nums'], item['url_object_id']
                        , item['front_image_url'], item['front_image_path'], item['praise_nums'], item['comment_nums']
                        , item['tags'], item['content']))

3.修改MysqlTwistedPipeline爲公用pipeline(防止多個pipeline去打開數據庫造成異常)

item.py中的修改，增加一個get_insert_sql()

class JobBoleArticleItem(scrapy.Item):
    title = scrapy.Field(
        input_processor=MapCompose(lambda x: x + "-jobbole")
        # output_processor=TakeFirst()
    )
    create_date = scrapy.Field()
    url = scrapy.Field()
    url_object_id = scrapy.Field()
    front_image_url = scrapy.Field(
        output_processor=MapCompose(return_value)
    )
    front_image_path = scrapy.Field()
    praise_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    fav_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    comment_nums = scrapy.Field(
        input_processor=MapCompose(get_nums)
    )
    tags = scrapy.Field(
        input_processor=MapCompose(remove_comment_tags),
        output_processor=Join(",")
    )
    content = scrapy.Field()

    def get_insert_sql(self):
        insert_sql = """
                                insert into jobbole_article(title,url,create_date,fav_nums,url_object_id,front_image_url,
                                        front_image_path,praise_nums,comment_nums,tags,content) 
                                values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);
                            """
        parsms = (self['title'], self['url'], self['create_date'], self['fav_nums'], self['url_object_id']
                  , self['front_image_url'], self['front_image_path'], self['praise_nums'], self['comment_nums']
                  , self['tags'], self['content'])
        return insert_sql,parsms

pipeline中的修改：

    def do_insert(self, cursor, item):
        # 執行具體的數據插入

        # 根據不同item構建不同的sql語句並插入到mysql中

        insert_sql, params = item.get_insert_sql()
        cursor.execute(insert_sql, params)

特別注意：port=3306不能是str類型