豆瓣排行榜小爬蟲

import requests
from lxml import etree

# 1. 將目標網站的信息抓取下來
headers = {
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 "
                  "Safari/537.36 ",
    'Referer': "https://movie.douban.com/"
}
url = "https://movie.douban.com/chart"
response = requests.get(url, headers=headers)
# print(response.text)
text = response.text


# 2. 將抓取下來的信息按一定的規則進行提取
html = etree.HTML(text)
trs = html.xpath("//tr[@class='item']")
movies = []
for tr in trs:
    # print(etree.tostring(tr, encoding='UTF-8').decode('UTF-8'))
    a_s = tr.xpath(".//a[@class='nbg']")    # xpath 永遠返回的是一個列表
    for a in a_s:
        title = a.xpath("@title")   # 電影名
        img = a.xpath(".//img//@src")   # 圖片
    divs = tr.xpath(".//div[@class='pl2']")
    for div in divs:
        actors = div.xpath(".//p[@class='pl']")
        for actor in actors:
            actor = actor.xpath(".//text()")    # 演員
        score = div.xpath(".//span[@class='rating_nums']//text()")  # 評分
    movie = {
        'title': title,
        'score': score,
        'img': img,
        'actor': actor,
    }
    movies.append(movie)
print(movies)

爬取頁面如下: