Python爬蟲（四）：新增縱橫中文網爬蟲Demo--爬取136書屋小說，並保存至本地文本文件中，單進程多進程對比效率（以三生三世十里桃花爲例）

---

運行環境：Python3.6

---

2019-05-24更新，由於原有的頁面改版了，所以現在新增了一個[縱橫中文網book.zongheng.com]採集代碼Demo

• 存在反爬，導致爬蟲運行出錯，下面兩個方法親測可以解決
  • 加入代理IP，我寫了一個代理IP提取接口 -> 跳轉；
  • 將瀏覽器訪問生成的Cookie信息加入到headers中；
• 該爬蟲不能正確抓取VIP權限才能訪問的內容

# -*- coding: utf-8 -*-
# @Author : Leo

import re
import os
import logging
import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter

logging.basicConfig(level=logging.INFO,  # 最低輸出
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S')


class ZonghengSpider:
    """
    縱橫中文網爬蟲
    - http://book.zongheng.com/
    """
    # 小說保存主路徑
    novel_save_dir = 'novels'
    session = requests.session()
    # 設置重試次數
    session.mount('http://', HTTPAdapter(max_retries=3))
    session.mount('https://', HTTPAdapter(max_retries=3))

    def __init__(self):
        self.session.headers.update(
            {'Host': 'book.zongheng.com',
             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) '
                           'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'})
        self.chapter_url = 'http://book.zongheng.com/api/chapter/chapterinfo?bookId={book_id}&chapterId={chapter_id}'

    def crawl(self, target_url: str):
        """
        開始爬取當前指定url
        :param target_url: 爲需要爬取的書籍頁面URL
        :return:
        """

        def request_url(url):
            resp = self.session.get(url=url)
            if resp.status_code == 200:
                return resp.json()
            else:
                return None

        book_name, book_id, chapter_id = self.get_page_info(target_url)
        logging.info(f'獲取到的書籍名: {book_name}, 書籍ID: {book_id}, 首章ID: {chapter_id}')
        if all([book_name, book_id, chapter_id]):
            # 設置保存路徑
            novel_save_path = os.path.join(self.novel_save_dir, book_name)
            if not os.path.exists(novel_save_path):
                os.makedirs(novel_save_path)
            logging.info(f'書籍保存路徑: {novel_save_path}')
            index = 0
            while True:
                index += 1
                chapter_url = self._get_chapter_url(book_id, chapter_id)
                logging.info(f'當前請求章節URL: {chapter_url}')
                chapter_json = request_url(url=chapter_url)
                if chapter_json is not None:
                    chapter_data = chapter_json.get('data')
                    if not chapter_data:
                        break
                    # 章節名
                    chapter_name = chapter_data.get('chapterName')
                    content_raw = chapter_data.get('content', '')
                    # 章節正文
                    clear_content = '\n'.join(
                        [repr(p).strip('\'') for p in BeautifulSoup(content_raw, 'html.parser').strings])
                    # TODO 作者、更新時間、章節字數...
                    with open(os.path.join(novel_save_path, str(index) + '-' + chapter_name + '.txt'), 'w',
                              encoding='utf8') as f:
                        f.write(clear_content)
                        logging.info('保存當前章節成功 > %s' % os.path.join(novel_save_path, str(index) + '-' + chapter_name))
                    # 設置下一章節的chapter_id
                    chapter_id = chapter_data.get('nexCid')
                else:
                    logging.error(f'請求章節URL異常, 異常URL: {chapter_url}')
            logging.info('採集完畢')

    def get_page_info(self, homepage_url):
        """
        獲取本書的book-id，以及首章的章節ID
        :param homepage_url: 書籍主頁url
        :return:
        """
        resp = self.session.get(url=homepage_url)
        if resp.status_code == 200:
            soup = BeautifulSoup(resp.text, 'html.parser')
            book_name = soup.find('div', {'class': 'book-name'}).get_text().strip()
            first_chapter_tag = soup.find('a', {'class': 'btn read-btn', 'href': True})
            if first_chapter_tag is not None:
                first_chapter_url = first_chapter_tag.get('href')
                result = re.findall(r'chapter/(\d+)/(\d+).html', first_chapter_url)
                book_id, chapter_id = result[0] if result else (None, None, None)
                return book_name, book_id, chapter_id
        else:
            logging.error('請求書籍主頁鏈接狀態嗎異常！')
        return None, None, None

    def _get_chapter_url(self, book_id, chapter_id):
        """
        獲取章節鏈接
        :param book_id:
        :param chapter_id:
        :return:
        """
        return self.chapter_url.format(book_id=book_id, chapter_id=chapter_id)


if __name__ == '__main__':
    spider = ZonghengSpider()
    spider.crawl(target_url='http://book.zongheng.com/book/840152.html')

簡介

• 小說網址：http://www.136book.com/
• 通過修改136book小說網中具體小說的url來爬取不同小說的分章節批量下載
• 該代碼以三生三世十里桃花爲例（鏈接）
• –>http://www.136book.com/sanshengsanshimenglitaohua/

運行效果展示
似乎圖掛掉了?

---

① book136_singleprocess.py

單進程保存小說章節

#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
# @Author : Woolei
# @File : book136_singleprocess.py

import requests
import time
import os
from bs4 import BeautifulSoup


headers = {
    'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
}


# 獲取小說章節內容，並寫入文本
def getChapterContent(each_chapter_dict):
    content_html = requests.get(each_chapter_dict['chapter_url'], headers=headers).text
    soup = BeautifulSoup(content_html, 'lxml')
    content_tag = soup.find('div', {'id': 'content'})
    p_tag = content_tag.find_all('p')
    print('正在保存的章節 --> ' + each_chapter_dict['name'])
    for each in p_tag:
        paragraph = each.get_text().strip()
        with open(each_chapter_dict['name'] + r'.txt', 'a', encoding='utf8') as f:
            f.write('  ' + paragraph + '\n\n')
            f.close()


# 獲取小說各個章節的名字和url
def getChapterInfo(novel_url):
    chapter_html = requests.get(novel_url, headers=headers).text
    soup = BeautifulSoup(chapter_html, 'lxml')
    chapter_list = soup.find_all('li')
    chapter_all_dict = {}
    for each in chapter_list:
        import re
        chapter_each = {}
        chapter_each['name'] = each.find('a').get_text()  # 獲取章節名字
        chapter_each['chapter_url'] = each.find('a')['href']  # 獲取章節url
        chapter_num = int(re.findall('\d+', each.get_text())[0])  # 提取章節序號
        chapter_all_dict[chapter_num] = chapter_each  # 記錄到所有的章節的字典中保存
    return chapter_all_dict


if __name__ == '__main__':
    start = time.clock()  # 記錄程序運行起始時間
    novel_url = 'http://www.136book.com/sanshengsanshimenglitaohua/'  # 這裏以三生三世十里桃花爲例
    novel_info = getChapterInfo(novel_url)  # 獲取小說章節記錄信息
    dir_name = '保存的小說路徑'
    if not os.path.exists(dir_name):
        os.mkdir(dir_name)
    os.chdir(dir_name)  # 切換到保存小說的目錄
    for each in novel_info:
        getChapterContent(novel_info[each])
        # time.sleep(1)
    end = time.clock()  # 記錄程序結束的時間
    print('保存小說結束，共保存了 %d 章，消耗時間：%f s' % (len(novel_info), (end - start)))
**在下載保存的過程中發現，消耗的時間比較長。所以計劃使用多進程來提高效率**

② book136_multiprocess.py

多進程保存小說章節

#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
# @Author : Woolei
# @File : book136_2.py 


import requests
import time
import os
from bs4 import BeautifulSoup
from multiprocessing import Pool

url = 'http://www.136book.com/huaqiangu/ebxeeql/'
headers = {
    'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
}


# 獲取小說章節內容，並寫入文本
def getChapterContent(each_chapter_dict):
    content_html = requests.get(each_chapter_dict['chapter_url'], headers=headers).text
    soup = BeautifulSoup(content_html, 'lxml')
    content_tag = soup.find('div', {'id': 'content'})
    p_tag = content_tag.find_all('p')
    print('正在保存的章節 --> ' + each_chapter_dict['name'])
    for each in p_tag:
        paragraph = each.get_text().strip()
        with open(each_chapter_dict['name'] + r'.txt', 'a', encoding='utf8') as f:
            f.write('  ' + paragraph + '\n\n')
            f.close()


# 獲取小說各個章節的名字和url
def getChapterInfo(novel_url):
    chapter_html = requests.get(novel_url, headers=headers).text
    soup = BeautifulSoup(chapter_html, 'lxml')
    chapter_list = soup.find_all('li')
    chapter_all_dict = {}
    for each in chapter_list:
        import re
        chapter_each = {}
        chapter_each['name'] = each.find('a').get_text()  # 獲取章節名字
        chapter_each['chapter_url'] = each.find('a')['href']  # 獲取章節url
        chapter_num = int(re.findall('\d+', each.get_text())[0])  # 提取章節序號
        chapter_all_dict[chapter_num] = chapter_each  # 記錄到所有的章節的字典中保存
    return chapter_all_dict


if __name__ == '__main__':
    start = time.clock()
    novel_url = 'http://www.136book.com/sanshengsanshimenglitaohua/'
    novel_info = getChapterInfo(novel_url)
    dir_name = '保存的小說路徑'
    if not os.path.exists(dir_name):
        os.mkdir(dir_name)
    os.chdir(dir_name)
    pool = Pool(processes=10)   # 創建10個進程
    pool.map(getChapterContent, [novel_info[each] for each in novel_info])
    pool.close()
    pool.join()
    end = time.clock()
    print('多進程保存小說結束，共保存了 %d 章，消耗時間：%f s' % (len(novel_info), (end - start)))

• 運行過程中，在任務管理器中可以看到創建了10個子進程(processes=10)，可以創建多個進程來提高效率，但是不考慮電腦性能而創建過多的進程後會使電腦執行效率嚴重下降。