DouyuSpider
今天学习了xpath方法, 拿斗鱼主播的房间信息做一个练习,有不足之处希望各位大佬帮忙斧正。
#encoding=utf-8
from selenium import webdriver
import time, json
class DouyuSpider:
def __init__(self):
self.start_url = "https://www.douyu.com/directory/all"
self.driver = webdriver.Chrome() # 实例化一个浏览器
self.page_num = 0 # 页数
def get_contnet_list(self):
# 先对li标签进行分组
li_list = self.driver.find_elements_by_xpath("//div[@class='layout-Module-container layout-Cover ListContent']//ul[@class='layout-Cover-list']/li")
content_list = []
# 再对每一组的li进行数据提取
for li in li_list:
item = {}
item["room_img"] = li.find_element_by_xpath(".//div[@class='DyListCover-imgWrap']//img").get_attribute("src")
item["room_title"]= li.find_element_by_xpath(".//h3[@class='DyListCover-intro']").text
item["room_cate"] = li.find_element_by_xpath(".//span[@class='DyListCover-zone']").text
item["anchor_name"]=li.find_element_by_xpath(".//h2[@class='DyListCover-user']").text
item["room_people"]=li.find_element_by_xpath(".//span[@class='DyListCover-hot']").text
content_list.append(item)
# 获取下一页的元素
next_page = self.driver.find_elements_by_xpath("//span[text()='下一页']")
next_page = next_page[0] if len(next_page) > 0 else None # 当执行到最后一页时, next_page为 None, 整个程序结束
return content_list, next_page
def save_content_list(self, content_list):
with open("douyu.txt", "a", encoding="utf-8") as f:
for content in content_list:
f.write(json.dumps(content, ensure_ascii=False))
f.write("\n\n")
self.page_num += 1
print("第{}页完毕".format(self.page_num))
# 实现主要逻辑
def run(self):
# 1.strat_url
# 2.发送请求
self.driver.get(self.start_url)
# 3.提取数据
content_list, next_page = self.get_contnet_list()
# 4.保存数据
self.save_content_list(content_list)
# 5.点击下一页元素
while next_page is not None:
next_page.click()
time.sleep(2) # 防止因执行速度过快,而导致浏览器没有接收到服务端发出的信息
content_list, next_page = self.get_contnet_list()
self.save_content_list(content_list)
if __name__ == '__main__':
douyu = DouyuSpider()
douyu.run()