今日頭條圖片的爬取

import re
import requests
import json
import os
from urllib import request

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}

url = 'https://www.toutiao.com/a6589933439766495747/'

response = requests.get(url,headers=headers)
html_str = response.text
# print(html_str)
pattern = r'gallery: JSON\.parse\((.*)\),'

match_res = re.search(pattern, html_str)

# 新建文件夾
if not os.path.exists('download') :
    os.mkdir('download')
if match_res:
    # 這本來就是str
    print(match_res.group(1))
    json_origin = match_res.group(1)
    # 這是第一遍loads, 返回值是str
    res_buzhidao = json.loads(json_origin)
    print(res_buzhidao)
    print(type(res_buzhidao))
    res_dict = json.loads(res_buzhidao)
    print(res_dict)
    print(type(res_dict))

    sub_images_list = res_dict['sub_images']
    for image in sub_images_list:
        image_url = image['url']
        filename = 'download/' + image_url.split('/')[-1] + '.jpg'
        # 下載圖片
        request.urlretrieve(image_url, filename)