鬥魚爬蟲 -- selenium技術

#!/usr/bin/env python
# -*- coding:utf-8 -*-

import time

from selenium import webdriver


chrome = webdriver.Chrome()

# invalid selector xpath語句錯誤
# 標題
# titles = chrome.find_elements_by_xpath('//h3[@class="ellipsis"]')
#
# # 關注量
# counts = chrome.find_elements_by_xpath('//span[@class="dy-num fr"]')
#
# print(titles)
# print(counts)
#
# print(len(titles),len(counts))
#
# for title in titles:
#     print(title.text)

fp = open('./鬥魚.txt',mode='a',encoding='utf-8')

# 第一頁內容
chrome.get('https://www.douyu.com/directory/all')
time.sleep(2)
# 將彈出來的對話框關閉
try:
    chrome.find_element_by_class_name('pop-zoom-hide').click()
except Exception as e:
    pass

while True:

    titles = chrome.find_elements_by_xpath('//h3[@class="ellipsis"]')
    counts = chrome.find_elements_by_xpath('//span[@class="dy-num fr"]')

    for title,count in zip(titles[10:],counts):
        fp.write(title.text.strip()+count.text.strip()+'\n')

#     給一個退出的條件
#     <a href="#" class="shark-pager-next shark-pager-disable shark-pager-disable-next">下一頁</a>
#     <a href="#" class="shark-pager-prev shark-pager-disable shark-pager-disable-prev">上一頁</a>
    index = chrome.page_source.find('shark-pager-disable-next')

    if index !=-1:
        break

#     點擊下一頁
    chrome.find_element_by_class_name('shark-pager-next').click()
    time.sleep(5)