使用scrapy抓取Youtube播放列表信息

可參看Knowsmore

抓取Youtube列表數據的前提是scrapy部署的機器可以正常訪問Youtube網站

示例網址

存取到Mongo中的數據如下：

{

"playlist_id" : "PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9",
"videos" : [
    {
        "playlist_id" : "PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9",
        "video_id" : "9pTwztLOvj4",
        "thumbnail" : [
            {
                "url" : "https://i.ytimg.com/vi/9pTwztLOvj4/hqdefault.jpg?sqp=-oaymwEZCPYBEIoBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLCmUXUPe-HgXiie0SRfL5cYz0JRrg",
                "width" : 245,
                "height" : 137
            }
        ],
        "title" : "Legend of the galactic heroes (1988) episode 1",
        "index" : 1,
        "length_seconds" : 1445,
        "is_playable" : true
    },
    {
        "playlist_id" : "PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9",
        "video_id" : "zzD1xU37Vtc",
        "thumbnail" : [
            {
                "url" : "https://i.ytimg.com/vi/zzD1xU37Vtc/hqdefault.jpg?sqp=-oaymwEZCPYBEIoBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLCnLCYaZVBeHnZR0T73rfEd_Dbyew",
                "width" : 245,
                "height" : 137
            }
        ],
        "title" : "Legend of the galactic heroes (1988) episode 2",
        "index" : 2,
        "length_seconds" : 1447,
        "is_playable" : true
    },

代碼如下：

# -*- coding: utf-8 -*-
import scrapy
import re
import json
from scrapy import Selector
from knowsmore.items import YoutubePlaylistItem, YoutubePlaylistVideoItem
from ..common import *

class YoutubeListSpider(scrapy.Spider):
    name = 'youtube_list'
    allowed_domains = ['www.youtube.com']
    start_urls = ['https://www.youtube.com/playlist?list=PLEbPmOCXPYV67l45xFBdmodrPkhzuwSe9']

    def parse(self, response):
        # Extract JSON Data with Regex Expression
        ytInitialData = r1(r'window\["ytInitialData"\] = (.*?)}};', response.body)
        if ytInitialData:
            ytInitialData = '%s}}' % ytInitialData
            ytInitialDataObj = json.loads(ytInitialData)

            # Assign VideoList info to variable
            playListInfo = ytInitialDataObj['contents']['twoColumnBrowseResultsRenderer']['tabs'][0]['tabRenderer']['content']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'][0]['playlistVideoListRenderer']

            # Build Scrapy Item
            playList = YoutubePlaylistItem(
                playlist_id = playListInfo['playlistId'],
                videos = []
            )

            # Insert the videoItem to YoutubePlaylistItem videos field
            for videoInfo in playListInfo['contents']:
                videoInfo = videoInfo['playlistVideoRenderer']
                videoItem = YoutubePlaylistVideoItem(
                    playlist_id = playListInfo['playlistId'],
                    video_id = videoInfo['videoId'],
                    thumbnail = videoInfo['thumbnail']['thumbnails'],
                    title = videoInfo['title']['simpleText'],
                    index = videoInfo['index']['simpleText'],
                    length_seconds = videoInfo['lengthSeconds'],
                    is_playable = videoInfo['isPlayable']
                )
                playList['videos'].append(videoItem)
            
            yield playList