python學習——爬蟲request異常代碼

一、網上摘錄的https://blog.csdn.net/xunkhun/article/details/79349621 

def getHTMLText(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "產生異常"

r.raise_for_status()會跑出異常，沒有異常則是None。抓取一切異常

二、自己用的，也是抄襲這位博文（https://blog.csdn.net/polyhedronx/article/details/81589196）

def get_text_from_url(url):
    # 從鏈接中獲取網頁文本
    try:
        response = requests.get(url, headers=headers, timeout=30)
        if response.status_code == 200:
            return response.text
        else:
            print('get page failed.', response.status_code)
            return None
    except (ConnectionError, TimeoutError) as e:
        print('crawling failed', url, e)
        return None