[Python爬蟲]使用Scrapy框架爬取圖蟲圖片

啓動文件main.py

from scrapy.cmdline import execute

execute('scrapy crawl tuchong'.split())

在spiders下的執行爬蟲文件

# -*- coding: utf-8 -*-
import scrapy,json
from ..items import tu_baoc   #實例化函數

class TuchongSpider(scrapy.Spider):
    name = 'tuchong'
    allowed_domains = ['stock.tuchong.com']
    # start_urls = ['https://tuchong.com/rest/2/sites/2634795/posts?count=20&page=1&before_timestamp=0']

    # 我們使用這個函數作爲初始的執行函數
    def start_requests(self):

        # 循環拼接頁數，這裏我循環了3次取了前三頁
        for i in range(1,4):
            url='https://tuchong.com/rest/2/sites/2634795/posts?count=20&page='+str(i)+'&before_timestamp=0'

            # 使用get方法請求，發送給parse
            req=scrapy.Request(url=url,callback=self.parse)

            # 發送
            yield  req

    def parse(self, response):

        # 調用items裏的自定義tu_baoc類實例化函數
        # 創建實例化對象
        t=tu_baoc()

        # 將返回值賦值給req
        req=response.text

        # 用json進行編碼
        html=json.loads(req)

        # 遍歷出post_list下的信息
        for i in html['post_list']:

            # 獲取名稱
            name=i['title']

            # 將遍歷圖片的鏈接和名稱
            for j in i['images']:
                # 獲取的是圖片的id，然後我將id拼接成url
                t['img_url']= 'http://photo.tuchong.com/2634795/f/{}.jpg'.format(j['img_id'])
                t['name']=name

                print(t['img_url'])
                # 我是將圖片下載到了本地也可以保存到數據庫裏等等
                # 將鏈接發送給pipelines.py的img_URL_baoc
                yield t

items.py

# 自定義實例化函數，構造函數
class tu_baoc(scrapy.Item):
    # define the fields for your item here like:
    # 創建參數
    img_url = scrapy.Field()
    name=scrapy.Field()

配置文件settings.py

#導包
from ..piaot import *

#ROBOTSTXT_OBEY改成False
ROBOTSTXT_OBEY = False

#開啓報頭
DEFAULT_REQUEST_HEADERS = {

  "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
}

#配置存儲文件地址和優先級
ITEM_PIPELINES = {
   'tu_chong.pipelines.img_URL_baoc': 300,
}

存儲文件pipelines.py

# -*- coding: utf-8 -*-
import requests,time        #調用時間函數time
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html


class TuChongPipeline(object):
    def process_item(self, item, spider):
        return item


class img_URL_baoc(object):
    def process_item(self, item, spider):

        # 將返回的值再用get請求返回二進制
        req=requests.get(url=item['img_url'])

        # 用時間函數當隨機名稱
        x = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))

        # 拼接本地地址
        url="C:/Users/黑神/Desktop/pc_zy/圖蟲/"+item['name']+x+'.jpg'

        # 保存圖片
        with open(url,"wb") as f:
            f.write(req.content)