爬蟲學習之05-request模塊獲取某百科下的全部糗圖

"""
    獲取某百科下的全部糗圖:https://www.***baike.com/imgrank/
    代碼僅供學習，請勿作其他用途
    date:2020-05-25
"""
import requests
import re
import os

if __name__ == '__main__':
    requests.packages.urllib3.disable_warnings()
    # UA僞裝
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36'
    }

    if not os.path.exists('05糗圖'):
        os.mkdir('05糗圖')

    # 1.指定URL
    url = 'https://www.***baike.com/imgrank/page/%d/'

    for pageNum in range(1,14):
        new_url = format(url%pageNum)
        #2.爬取頁面
        page_data = requests.get(url=new_url, headers=headers, verify=False).text
        # print(page_data)

        #3. 使用聚焦爬蟲對頁面中所有的糗圖進行解析
        """
        <div class="thumb">
        
        <a href="/article/123144519" target="_blank">
        <img src="//pic.***baike.com/system/pictures/12314/123144519/medium/763CX35F1320P2G1.jpg" alt="糗事#123144519" class="illustration" width="100%" height="auto">
        </a>
        </div>
        """
        # 用正則解析
        ex = '<div class="thumb">.*?<img src="(.*?)" alt=.*?</div>'
        img_src_list = re.findall(ex, page_data, re.S)

        for src in img_src_list:
            src = 'https:' + src
            img_data = requests.get(url=src, headers=headers, verify=False).content
            # 生成圖片名稱
            img_name = src.split('/')[-1]
            imgpath = './05糗圖/' + img_name

            with open(imgpath, 'wb') as fp:
                fp.write(img_data)
                print(img_name, '下載完成')

運行結果如下：