bs4解析器——爬取三國演義目錄和內容

'''
爬取三國演義的目錄和內容
'''
import requests
from bs4 import BeautifulSoup

if __name__ == '__main__':
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'
    }
    url = "http://www.shicimingju.com/book/sanguoyanyi.html"
    response = requests.get(url=url,headers = headers)
    page_text = response.text
    #構建soup對象
    soup = BeautifulSoup(page_text,'lxml')
    #層級選擇器，找到li標籤
    li_list = soup.select(".book-mulu > ul >li")
    fp = open('三國演義.txt', 'w', encoding='utf-8')
    for li in li_list:
        title = li.a.string#獲取a標籤內的直接文本內容
        detail_url = "http://www.shicimingju.com"+ li.a['href']#獲取a標籤的href屬性值
        detail_page = requests.get(url = detail_url,headers=headers).text
        detail_soup = BeautifulSoup(detail_page,'lxml')
        div_tag = detail_soup.find("div",class_='chapter_content')#查找第一個類名爲xx的div
        content = div_tag.text#獲取div標籤內的全部文本內容
        fp.write(title+':'+content+'\n')
        print(title,'爬取成功')