python爬蟲(四）——模擬登錄丁香園

實戰

實戰小項目：模擬登錄丁香園，抓取論壇頁面人員基本信息與回覆帖子。丁香園論壇：http://www.dxy.cn/bbs/thread/626626#626626

思路

1. 首先把登陸方式由掃碼切換爲賬號密碼登陸

js1 = 'document.querySelector("#j_loginTab1").style.display="none";'
browser.execute_script(js1)
time.sleep(1)
js2 = 'document.querySelector("#j_loginTab2").style.display="block";'
browser.execute_script(js2)

2. 定位到賬號，密碼位置並輸入自己的帳號密碼，點擊登錄

input_name = browser.find_element_by_name('username')
input_name.clear()
input_name.send_keys(********')
input_pass = browser.find_element_by_name('password')
input_pass.clear()
input_pass.send_keys('********')
browser.find_element_by_xpath('//*[@class="form__button"]/button').click()

3. 抓取頁面，利用xpath定位到抓取內容的位置

user = tree.xpath('//div[@id="postcontainer"]//div[@class="auth"]/a/text()')
content = tree.xpath('//td[@class="postbody"]')

完整代碼

# -*- coding:utf-8 -*-
import requests, json, re, random,time
from bs4 import BeautifulSoup
from selenium import webdriver
from lxml import etree

class getUrl(object):
    """docstring for getUrl"""
    def __init__(self):
        self.headers={
            "Connection": "keep-alive",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "  
                          "(KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate, sdch",
            "Accept-Language": "zh-CN,zh;q=0.8"
        };
    def run(self):
        browser = webdriver.Chrome()
        browser.get('https://auth.dxy.cn/accounts/login?service=http://www.dxy.cn/bbs/index.html')
        time.sleep(1)
        # 切換賬號密碼登錄表單
        js1 = 'document.querySelector("#j_loginTab1").style.display="none";'
        browser.execute_script(js1)
        time.sleep(1)
        js2 = 'document.querySelector("#j_loginTab2").style.display="block";'
        browser.execute_script(js2)
        #輸入賬號密碼
        input_name = browser.find_element_by_name('username')
        input_name.clear()
        input_name.send_keys('*********')
        input_pass = browser.find_element_by_name('password')
        input_pass.clear()
        input_pass.send_keys('*******')
        browser.find_element_by_xpath('//*[@class="form__button"]/button').click()
        #此步驟應該有驗證碼，先跳過
        time.sleep(10)
        cookie = browser.get_cookies()
        cookie_dict = {i['name']:i['value'] for i in cookie}
        #轉到抓取頁面
        browser.get("http://www.dxy.cn/bbs/thread/626626#626626");
        html = browser.page_source
        tree = etree.HTML(html)
        user = tree.xpath('//div[@id="postcontainer"]//div[@class="auth"]/a/text()')
        content = tree.xpath('//td[@class="postbody"]')
        for i in range(0,len(user)):
            result = user[i].strip()+":"+content[i].xpath('string(.)').strip()
            #寫入文件
            dir_file = open("DXY_records.txt",'a', encoding="utf-8")
            dir_file.write(result+"\n")
            dir_file.write('*' * 80+"\n")
            dir_file.close()
        print('*' * 5 +"抓取結束"+'*' * 5)


if __name__ == '__main__':
    geturl = getUrl()
    geturl.run()