middlewares.py編寫
from p5.settings import UserAgent_list
import random
class RandomUserAgentMiddleware(object):
#當每個request請求發送給目標網站時調用，用於處理或修改request對象
def process_request(self, request, spider):
ua = random.choice(UserAgent_list)
request.headers["User-Agent"] = ua
return None
class CheckUserAgent(object):
#，在將request獲取到響應傳遞給引擎之前調用，在將響應傳遞給引擎之前處理收到的響應，如：響應失敗重新請求，或將失敗的做一定處理再返回給引擎
def process_response(self, request, response, spider):
print(request.headers["User-Agent"])
return response
在settings.py中開啓下載器中間件
DOWNLOADER_MIDDLEWARES = {
'p5.middlewares.RandomUserAgentMiddleware': 543,
'p5.middlewares.CheckUserAgent': 544,
}
常見的User-Agent列表
USER_AGENT_LIST = [“User-Agent,Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;”,“User-Agent,Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)”,“User-Agent,Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)”,“User-Agent, Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)”,“User-Agent, Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1”,“User-Agent,Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1”,“User-Agent,Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11”,“User-Agent,Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11”,“User-Agent,Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50”,“User-Agent, Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)”,“User-Agent, Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)”,“User-Agent, Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)”]