Pyton爬蟲實現豆瓣登陸

• 目標：實現豆瓣登陸；
• 方法：使用requests.Session 來處理cookies
• 問題：驗證碼輸入

關於headers

1.實現登陸

# -*- coding: utf-8 -*-
import requests
from HTMLParser import HTMLParser
class DoubanClient(object):
    def __init__(self):
        object.__init__(self)
        #在chrome瀏覽器進行登陸，登陸之後：右鍵 檢查-Newwork-login-Headers,可以查看
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
            'Origin': 'https://accounts.douban.com'}
        self.session = requests.session()  # 創建一個session
        self.session.headers.update(headers)  # 對頭進行定製

    def login(self, usename, pwd, source='index_nav', redir='https://www.douban.com', user_login='登陸'):
        #下面的地址可能有變化，根據實際進行更改
        url = 'https://accounts.douban.com/login'  # 訪問登陸的主頁面，爲了獲取驗證碼id和驗證碼圖片地址
        r = self.session.get(url)
        (captcha_id, captcha_url) = _get_captcha(r.content)  # 解析網頁內容，從這個函數中獲取驗證碼
        if captcha_id:
            captcha_solution = raw_input('Please input solution for captcha[%s]:' % captcha_url)  # 用戶輸入驗證
        data = {'source': source,
                'form_email': usename,
                'form_password': pwd,
                'redir': redir,  # 跳轉地址
                'user_login': user_login,
                }
        if captcha_id:
            data['captcha-id'] = captcha_id
            data['captcha-solution'] = captcha_solution

        headers = {'Referer': 'https://accounts.douban.com/login',
                   'Host': 'accounts.douban.com'}

        self.session.post(url, data=data, headers=headers)
        print(self.session.cookies.items())
 #下面這個函數用在解析html文件中，tag匹配
def _attr(attrs, attrname):
    for attr in attrs:
        if attr[0] == attrname:
            return attr[1]
    return attr[0]
def _get_captcha(content):
    class CaptchaParser(HTMLParser):
        def __init__(self):
            HTMLParser.__init__(self)
            self.captcha_id = None
            self.captcha_url = None
        def handle_starttag(self, tag, attrs):
            if tag == 'img' and _attr(attrs, 'id') == 'captcha_image' and _attr(attrs, 'class') == 'captcha_image':
                self.captcha_url = _attr(attrs, 'src')  # 如果是url = src
            if tag == 'input' and _attr(attrs, 'type') == 'hidden' and _attr(attrs, 'name') == 'captcha-id':
                self.captcha_id = _attr(attrs, 'value')

    p = CaptchaParser()
    p.feed(content)
    return p.captcha_id, p.captcha_url
def _get_ck(content):
    class CKParser(HTMLParser):
        def __init__(self):
            HTMLParser.__init__(self)
            self.ck = None
        def handle_starttag(self, tag, attrs):
            if tag == 'input' and _attr(attrs, 'type') == 'hidden' and _attr(attrs, 'name') == 'ck':
                self.ck = _attr(attrs, 'value')

    p = CKParser()
    p.feed(content)
    return p.ck
c = DoubanClient()
c.login('[email protected]', 'dashab123')

注：其中登陸的地址可能發生改變，另外注意大小寫，一切需要根據看到的headers