python3 爬蟲日記(三) 爬取堆糖動態加載網頁

1.分析：進入堆糖網後我們在分類找到插畫繪畫進入這個分類後發現好多圖片，下拉後發現會有不斷的圖片刷新出來，這就是堆糖採用了動態加載網頁。

2.用開發者工具(F12)分析：按一下F12，找到network分支，再按一下F5，將刷新後的網頁一直往下拉，打開XHR，發現Name下有兩個或多個？include開頭字段，然後觀察Header和Preview發現它的圖片信息是json格式的數據。

3.準備開工。

# -*- coding:utf-8 -*-
import pymongo
from requests.exceptions import RequestException
import requests
import json
from urllib.parse import urlencode

def get_index_page(start_page,id_page):
    headers = {
        'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
        'Referer':'http://www.duitang.com/category/?cat=painting',
        'Accept': 'text/plain, */*; q=0.01',
        'Host': 'www.duitang.com',
        'Accept - Encoding': 'gzip,deflate, sdch',
        'Accept - Language': 'zh - CN, zh;q=0.8',
        'Connection': 'keep - alive',
    }
    data = {
        'include_fields': 'top_comments,'
        'is_root, source_link, item, buyable, root_id, status, like_count, sender, album',
        'filter_id': '插畫繪畫',
        'start':start_page,
        '_':id_page,
    }
    url = 'http://www.duitang.com/napi/blog/list/by_filter_id/?' + urlencode(data)	# 拼接URL
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('請求索引頁出錯!')
        return None

def parse_page_index():
    i = 0
    n = 1498560199148
    while i < 23976:
        i = i + 24
        n = n + 1
        html = get_index_page(i, n)
        for n in range(24):
            data = json.loads(html.strip())		# 將json字典轉換爲python字典
            img_url = data['data']['object_list'][n]['photo']['path']	# 獲取字典中的圖片鏈接
            title = data['data']['object_list'][n]['msg']		# 獲取字典中的標題
            post_sub.insert_one({'img_id': title, 'img_url': img_url})	# 插入到數據庫中
            print(title,img_url)

if __name__ == '__main__':
    connection = pymongo.MongoClient()
    post_info = connection.duitang_painting
    post_sub = post_info.duitang
    parse_page_index()