Scrapy 循环顺序运行多个任务

基于 Scrapy 框架扩展，循环顺序执行多个任务

# -*- coding: utf-8 -*-

from scrapy.cmdline import execute
import sys
import os
import time


def run():
    # 根据业务需求自定义设置每个任务运行时间，CLOSESPIDER_TIMEOUT
    while 1:
        os.system("scrapy crawl py1 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py2 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py3 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py4 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py5 -s CLOSESPIDER_TIMEOUT=1200")
        os.system("scrapy crawl py6 -s CLOSESPIDER_TIMEOUT=1200")

run()