Scrapy 循環順序運行多個任務

基於 Scrapy 框架擴展，循環順序執行多個任務

# -*- coding: utf-8 -*-

from scrapy.cmdline import execute
import sys
import os
import time


def run():
    # 根據業務需求自定義設置每個任務運行時間，CLOSESPIDER_TIMEOUT
    while 1:
        os.system("scrapy crawl py1 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py2 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py3 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py4 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py5 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py6 -s CLOSESPIDER_TIMEOUT=1200")

run()