圖片爬蟲

參考視頻學習，堆糖網圖片爬蟲

"""
1.URL
2.模擬瀏覽器請求資源
3.解析網頁
4.保存數據到本地
"""
import requests   #第三方庫
import urllib.parse
import json
import jsonpath

url ='https://www.duitang.com/napi/blog/list/by_search/?kw={}&start={}'
label = '校花'
label = urllib.parse.quote(label)
#print(label)
num = 0

for index in range(0, 2400, 24):
    u = url.format(label, index)
    we_data = requests.get(u).text
#    print(we_data)
    html = json.loads(we_data)
    photo = jsonpath.jsonpath(html, "$..path")
    print(photo)
    for i in photo:
        a = requests.get(i)
        with open(r'G:\Python\test\圖片爬蟲\photo\{}.jpg'.format(num), 'wb') as f:
            f.write(a.content)
        num += 1

 

 以下爲自己結合小說爬蟲和圖片爬蟲，對一個圖片網站進行的爬取。圖片不雅，故網址做了屏蔽，僅供代碼學習

import requests
from pyquery import PyQuery

photo = []
num = 3538

def onepage(one_url, oneflag):
    response = requests.get(url=one_url)
    if response.status_code != 200:
        return False
    #print(response.text)
    doc = PyQuery(response.text)
    title = doc('title').text()
    print(title)
    desc = doc('head > meta:nth-child(6)').attr('content')
    print(desc)

    imglist = doc('#main > article > div > p a')
    #print(imglist)
    for dd in imglist.items():
        photo.append(dd.attr('href'))
#        print(dd.attr('href'))

    if oneflag == True:
        with open(r'G:\Python\test\圖片爬蟲\photo3\說明.txt', encoding='utf-8', mode='a+') as f1:
            f1.write(title+'\n')
        page = doc('#main > article > div > div.page-links a')
        for i in page.items():
            with open(r'G:\Python\test\圖片爬蟲\photo3\說明.txt', encoding='utf-8', mode='a+') as f1:
                f1.write(i.attr('href')+'\n')
            print(i.attr('href'))
            onepage(i.attr('href'), False)
        with open(r'G:\Python\test\圖片爬蟲\photo3\說明.txt', encoding='utf-8', mode='a+') as f1:
            f1.write('\n\n\n')


for j in range(143, 1000):
    url = 'https://******.com/blog/archives/'+str(j)
    photo = []
    if onepage(url, True)==False:
        continue
    print(j)
    print(photo)
    for i in photo:
        a = requests.get(i)
        with open(r'G:\Python\test\圖片爬蟲\photo3\{}-{}.jpg'.format(j, num), 'wb') as f2:
            f2.write(a.content)
        num += 1