網絡爬蟲 | 貓眼電影最受期待榜信息抓取 —— 基於multiprocessing / re（多進程/正則表達式）

import re
import requests
from requests.exceptions import RequestException
import multiprocessing
import random


headers = {'User-Agent':  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16"}
def get_one_page(url):
    """
    獲取當前頁的html文本信息
    :param url:
    :return:
    """
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            # 此處要進行重新編碼
            return response.text
        return None
    except RequestException as e:
        return None


def parse_one_page(html):
    pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?<a.*?data-val.*?>(.*?)</a>.*?<p .*?start">(.*?)</p>.*?'
                         '<p.*?releasetime">(.*?)</p>', re.S)
    items = re.findall(pattern, html)
    for i in items:
        yield {
            "index": items[0],
            "title": items[1],
            "star": items[2],
            "releasetime": items[3]
        }


def main(offset):
    url = "https://maoyan.com/board/6?offset=%s"
    html = get_one_page(url % offset)
    print([item for item in parse_one_page(html)])


if __name__ == "__main__":
    pool = multiprocessing.Pool(4)
    for i in range(10):
        pool.apply_async(main, (i*10, ))

    pool.close()
    pool.join()