xpath練習

DouyuSpider

今天學習了xpath方法， 拿鬥魚主播的房間信息做一個練習，有不足之處希望各位大佬幫忙斧正。

#encoding=utf-8

from selenium import webdriver
import time, json

class DouyuSpider:

    def __init__(self):
        self.start_url = "https://www.douyu.com/directory/all"
        self.driver    = webdriver.Chrome() # 實例化一個瀏覽器
        self.page_num  = 0                  # 頁數

    def get_contnet_list(self):
        # 先對li標籤進行分組
        li_list = self.driver.find_elements_by_xpath("//div[@class='layout-Module-container layout-Cover ListContent']//ul[@class='layout-Cover-list']/li")
        content_list = []
        # 再對每一組的li進行數據提取
        for li in li_list:
            item = {}
            item["room_img"] = li.find_element_by_xpath(".//div[@class='DyListCover-imgWrap']//img").get_attribute("src")
            item["room_title"]= li.find_element_by_xpath(".//h3[@class='DyListCover-intro']").text
            item["room_cate"] = li.find_element_by_xpath(".//span[@class='DyListCover-zone']").text
            item["anchor_name"]=li.find_element_by_xpath(".//h2[@class='DyListCover-user']").text
            item["room_people"]=li.find_element_by_xpath(".//span[@class='DyListCover-hot']").text
            content_list.append(item)
        # 獲取下一頁的元素
        next_page = self.driver.find_elements_by_xpath("//span[text()='下一頁']")
        next_page = next_page[0] if len(next_page) > 0 else None    # 當執行到最後一頁時, next_page爲 None, 整個程序結束
     	return content_list, next_page

    def save_content_list(self, content_list):
        with open("douyu.txt", "a", encoding="utf-8") as f:
            for content in content_list:
                f.write(json.dumps(content, ensure_ascii=False))
                f.write("\n\n")
        self.page_num += 1
        print("第{}頁完畢".format(self.page_num))

    # 實現主要邏輯
    def run(self):
        # 1.strat_url
        # 2.發送請求
        self.driver.get(self.start_url)
        # 3.提取數據
        content_list, next_page = self.get_contnet_list()
        # 4.保存數據
        self.save_content_list(content_list)
        # 5.點擊下一頁元素
        while next_page is not None:
            next_page.click()
            time.sleep(2)             # 防止因執行速度過快，而導致瀏覽器沒有接收到服務端發出的信息
            content_list, next_page = self.get_contnet_list()
            self.save_content_list(content_list)

if __name__ == '__main__':
    douyu = DouyuSpider()
    douyu.run()