from urllib import request,parse
from urllib.error import HTTPError,URLError
#從下面的urlrequest詳細封裝了post方法函數
def post(url,form=None,headers=None):
return urlrequests(url,form,headers)
#從下面的urlrequest詳細封裝了get方法函數
def get(url,headers=None):
return urlrequests(url,headers=headers)
#定義總方法
def urlrequests(url,form=None,headers=None):
html_bytes = b''
#自定義請求名字
user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
if not headers:
headers = {
'User-Agent':user_agent
}
try:
#判斷是否給他傳遞了form參數
if form:
#把form裝換成字符串
form_str = parse.urlencode(form)
#把其轉換成bytes
form_bytes = form_str.encode('utf-8')
#調用函數
response = request.urlopen(url,data=form_bytes)
else:
req = request.Request(url,headers=headers)
response = request.urlopen(req)
html_bytes = response.read()
#異常捕獲
except HTTPError as e:
print(e)
except URLError as e:
print(e)
return html_bytes
if __name__ == '__main__':
# url = 'http://www.baidu.com'
url = 'http://fanyi.baidu.com/sug'
# res = get(url)
# with open('baidudu.html','wb') as e:
# e.write(res)
form={
'kw':'你好'
}
res = post(url,form)
print(res)
爬蟲簡單封裝post和get方法
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.