1.打開middlewares.py文件
2.添加代理類
import base64
""" 阿布雲代理配置"""
proxy_server = "http://http-dyn.abuyun.com:9020"
proxy_user = "HEWGR9329K68Z11D"
proxy_pass = "ACEBFFCDFE62615F"
proxy_auth = "Basic " + base64.urlsafe_b64encode(bytes((proxy_user + ":" + proxy_pass), "ascii")).decode("utf8")
class ABYProxyMiddleware(object):
""" 阿布雲代理中間件 """
def process_request(self, request, spider):
request.meta["proxy"] = proxy_server
request.headers["Proxy-Authorization"] = proxy_auth
3.打開settings.py文件開啓中間件：
DOWNLOADER_MIDDLEWARES = {
'Securities.middlewares.ABYProxyMiddleware': 1,
}
4.阿布雲每秒請求次數有限制，可以在settings.py中對scrapy請求速度進行設置
""" 啓用限速設置 """
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 0.2 # 初始下載延遲
DOWNLOAD_DELAY = 0.2 # 每次請求間隔時間
