Scrapy使用MySQL

供以後寫代碼參考：

一、同步執行

import pymysql


class CollectDataPipeline(object):
    def open_spider(self, spider):
        print('open' * 20)
        self.conn = pymysql.connect(host="127.0.0.1", user="root", password="****", database="collect_data",
                                    charset="utf8")
        self.db_cur = self.conn.cursor()

    def close_spider(self, spider):
        self.conn.close()

    def process_item(self, item, spider):
        val = (item['title'],
               item['contents'],
               item['time'])
        # 先刪除數據
        sql = '''delete from data where length(time)>0;'''
        self.db_cur.execute(sql)
        self.conn.commit()
        # 插入新數據
        sql = '''INSERT INTO data(title,contents,time) VALUES(%s,%s,%s);'''
        self.db_cur.execute(sql, val)
        self.conn.commit()
        return item

二、異步執行

import pymysql
from twisted.enterprise import adbapi
# 異步更新操作
class LvyouPipeline(object):
    def __init__(self, dbpool):
        self.dbpool = dbpool
 
    @classmethod
    def from_settings(cls, settings):  # 函數名固定，會被scrapy調用，直接可用settings的值
        """
        數據庫建立連接
        :param settings: 配置參數
        :return: 實例化參數
        """
        adbparams = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            password=settings['MYSQL_PASSWORD'],
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor   # 指定cursor類型
        )
 
        # 連接數據池ConnectionPool，使用pymysql或者Mysqldb連接
        dbpool = adbapi.ConnectionPool('pymysql', **adbparams)
        # 返回實例化參數
        return cls(dbpool)
 
    def process_item(self, item, spider):
        """
        使用twisted將MySQL插入變成異步執行。通過連接池執行具體的sql操作，返回一個對象
        """
        query = self.dbpool.runInteraction(self.do_insert, item)  # 指定操作方法和操作數據
        # 添加異常處理
        query.addCallback(self.handle_error)  # 處理異常

    #其他代碼都一樣，SQL語句不一樣即可，可複用。 
    def do_insert(self, cursor, item):
        val=(item['Name'], item['Address'], item['Grade'], item['Score'],item['Price'])
        # 對數據庫進行插入操作，並不需要commit，twisted會自動commit
        insert_sql = """
        insert into daname(name1, address, grade, score, price) VALUES (%s,%s,%s,%s,%s)
        """
        self.cursor.execute(insert_sql, val)
 
    def handle_error(self, failure):
        if failure:
            # 打印錯誤信息
            print(failure)