python併發模塊之concurrent.futures(二)
上次我們簡單的瞭解下，模塊的一些基本方法和用法，這裏我們進一步對concurrent.futures做一個瞭解和拓展.
上次的內容點這。
python併發模塊之concurrent.futures(二)
以下載圖片爲例子，下面的程序是順序下載http://www.58pic.com/newpic/28660111.html網站的24個表情 。
from requests_html import HTMLSession
import os
import time
BASE_PATH="downloads"
class Get_Image():
def __init__(self):
self.timeout=20
self.session=HTMLSession()
def getiamge(self,url):
req=self.session.get(url,timeout=self.timeout)
if req.status_code==200:
imgurllist=req.html.xpath("//ul[@class='emoticon-model']/li/img/@data-big")
for index,url in enumerate(imgurllist):
print(f"開始下載第{index+1}張圖片")
self.save_image(url,index+1)
else:
print("下載失敗")
def save_image(self,imgurl,index):
print(f"當前下載鏈接：{imgurl}")
buff=self.session.get(imgurl,timeout=self.timeout).content
file_path=os.path.join(os.path.dirname(os.path.abspath(__file__)),BASE_PATH)
if not os.path.exists(file_path):
os.makedirs(file_path)
with open(os.path.join(file_path,f"{index}.png"),"wb") as fs:
fs.write(buff)
if __name__ == '__main__':
start_url="http://www.58pic.com/newpic/28660111.html"
start=time.time()
Get_Image().getiamge(start_url)
end=time.time()
print(f"順序下載24張圖片用時:{end-start}")
#運行了兩次結果分別爲
#順序下載24張圖片用時:14.926000356674194
#順序下載24張圖片用時:14.07800030708313
使用concurrent.futures修改成併發之後
from requests_html import HTMLSession
import os
import time
from concurrent.futures import ThreadPoolExecutor
BASE_PATH="downloads"
MAX_WORKERS = 10 #最多使用10個線程
class Get_Image():
def __init__(self):
self.timeout=20
self.session=HTMLSession()
def getiamge(self,url):
req=self.session.get(url,timeout=self.timeout)
if req.status_code==200:
imgurllist=req.html.xpath("//ul[@class='emoticon-model']/li/img/@data-big")
works=min(len(imgurllist),MAX_WORKERS)
with ThreadPoolExecutor(works) as excutor:
res=excutor.map(self.save_image,imgurllist,range(1,25))
return len(list(res))
else:
print("下載失敗")
def save_image(self,imgurl,index):
print(f"當前下載鏈接：{imgurl}")
buff=self.session.get(imgurl,timeout=self.timeout).content
file_path=os.path.join(os.path.dirname(os.path.abspath(__file__)),BASE_PATH)
if not os.path.exists(file_path):
os.makedirs(file_path)
with open(os.path.join(file_path,f"{index}.png"),"wb") as fs:
fs.write(buff)
if __name__ == '__main__':
start_url="http://www.58pic.com/newpic/28660111.html"
start=time.time()
Get_Image().getiamge(start_url)
end=time.time()
print(f"併發下載24張圖片用時:{end-start}")
#運行了兩次結果分別爲
#併發下載24張圖片用時:7.737000226974487
#併發下載24張圖片用時:7.083999872207642
通過觀察發現速度併發之後效率大大提高了。