import requests
import re
import os
import time
"""獲取主網頁"""
web_page = 'https://www.vmgirls.com/'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
urls_response = requests.get(web_page,headers=headers)
urls_html = urls_response.text
"""解析主主網頁獲取下一層網頁"""
all_urls = re.findall('https://.*?/\d*.html',urls_html)
urls = list(set(all_urls))
# print(urls)
"""下載下一頁的網頁圖片"""
num_list = []
for url in urls:
url_resopnse = requests.get(url,headers=headers)
html=url_resopnse.text
dir_name = re.findall('<h1 class="post-title h3">(.*?)</h1>',html)[-1]
wget_urls = re.findall('https:.*?.jpeg',html)
print("\033[32;1m %s upload %s pictures\033[0m" %(dir_name,len(wget_urls)))
num = len(wget_urls)
num_list.append(num)
for wget_url in wget_urls:
time.sleep(1)
file_name = wget_url.split('/')[-1]
print(file_name)
dir_name = re.findall('<h1 class="post-title h3">(.*?)</h1>',html)[-1]
if not os.path.exists(dir_name):
os.mkdir(dir_name)
response = requests.get(wget_url,headers=headers)
with open(dir_name + '/' + file_name,'wb') as f:
f.write(response.content)
sum = 0
for i in num_list:
sum += i
print("\033[31;1mThere are %s pictures that need to be crawled\033[0m" % (sum))
python分頁爬取網站的圖片
發表評論
所有評論
還沒有人評論，想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.
相關文章
Shell/Python中的用戶名獲取
一、幾個基本概念 登錄用戶（login user）：通過登錄方式進入系統的用戶，強調登錄身份。 當前用戶（current user）：執行一個進程或者命令時所使用的用戶身份，強調執行身份。 舉