Python 多線程處理框架

參考：https://www.jianshu.com/p/8efdc122233e

多線程通用任務處理型驅動框架
probe_type 探測類型rtsp或者http
task_queue 任務隊列
task_handler 任務處理函數
thread_count 線程數數目
result_queue 結果存放隊列
args,kwargs爲可變參數列表，爲擴展性考慮

# coding=utf-8
# version:python3.6.0

# class TaskHandler:
#     """
#     任務接口類，用於擴展任務的基礎類
#     """
#     # 當前系統中的任務
#     # 按照{taskid, task_handler}的方式存儲
#     registered_tasks = None
#     # 系統是否初始化，一般用於文件夾和系統參數的初始化
#     initialized = False
#
#     def __init__(self, **kwargs):
#         self.service_enabled = False
#         self.registered_tasks = {}
#
#     def start_service_from_pause(self, taskid):
#         """
#         任務從暫停到啓動
#         :param taskid: 任務ID
#         :type Biginteger
#         :return:
#         """
#
#     def pause_service(self, taskid):
#         """
#         暫停指定ID的任務
#         :param taskid: 任務ID
#         :return:
#         """
#
#     def start(self, taskid):
#         """
#         啓動指定id的任務
#         :param taskid: 任務ID
#         :return:
#         """

import queue
import argparse
import threading
import time

class MultiThreadHandler(object):
    """
    多線程通用任務處理型驅動框架
    task_queue 任務隊列
    task_handler 任務處理函數
    thread_count 線程數數目
    result_queue 結果存放隊列
    args,kwargs爲可變參數列表，爲擴展性考慮
    """

    def __init__(self, task_queue, task_handler, result_queue=None, thread_count=1, *args, **kwargs):
        self.task_queue = task_queue
        self.task_handler = task_handler
        self.result_queue = result_queue
        self.thread_count = thread_count
        self.args = args
        self.kwagrs = kwargs
        self.thread_pool = []

    def run(self, block_flag):
        for i in range(self.thread_count):
            t = _TaskHandler(self.task_queue, self.task_handler, self.result_queue, *self.args, **self.kwagrs)
            self.thread_pool.append(t)
        for th in self.thread_pool:
            th.setDaemon(True)
            th.start()
        '''
        # 阻塞等待所有線程結束
        if block_flag:
            for th in thread_pool:
                threading.Thread.join(th)
        '''
        # 阻塞等待所有線程結束
        while self._check_stop():
            try:
                time.sleep(1)
            except KeyboardInterrupt:
                print('KeyboardInterruption')
                self.stop_all()
                break
        print('>>>all Done')

    def _check_stop(self):
        """檢查線程池中所有線程是否全部運行完"""
        finish_num = 0
        for th in self.thread_pool:
            if not th.isAlive():
                finish_num += 1

        return False if finish_num == len(self.thread_pool) else True

    def stop_all(self):
        """掉用線程體stop方法，停止所有線程"""
        for th in self.thread_pool:
            th.stop()

class _TaskHandler(threading.Thread):
    """
    一個任務處理器線程，task_queue任務隊列,task_handler任務處理函數,result_queue是結果隊列，args,kwargs可變控制參數
    可外部中斷
    """

    def __init__(self, task_queue, task_handler, result_queue=None, *args, **kwargs):
        threading.Thread.__init__(self)
        self.task_queue = task_queue
        self.task_handler = task_handler
        self.result_queue = result_queue
        self.args = args
        self.kwargs = kwargs
        self.is_stoped = True

    def run(self):
        while self.is_stoped:
            try:
                item = self.task_queue.get(False)  # block= False
                self.task_handler(item, self.result_queue, *self.args, **self.kwargs)
                self.task_queue.task_done()  # 退出queue
            except queue.Empty as e:
                print("all task has done!")
                break
            except Exception as e:
                print("error:", e)
                # time.sleep(1)

    def stop(self):
        self.is_stoped = False

def out(item, result_queue):  # 加載處理函數
    result_queue.put(item)

if __name__ == '__main__':
    # parse the command args
    start = time.time()
    parse = argparse.ArgumentParser()
    parse.add_argument("-f", "--file", default='yyqtext.txt',help="the target file")
    parse.add_argument("-th", "--thread", type=int, default=1, help="the thread number")
    parse.add_argument("-o", "--outfile", default='yyqout.txt',help="the outputfile")
    # 解析命令行
    results = parse.parse_args()
    filename = results.file
    th = results.thread
    outfile = results.outfile
    task_queue = queue.Queue()
    out_queue = queue.Queue()
    with open(filename, "r+") as f:
        for line in f:
            line = line.rstrip()
            if line:
                task_queue.put(line)

    MultiThreadHandler(task_queue, out, out_queue, th).run(True)

    with open(outfile, "w+") as f:
        while True:
            f.write(out_queue.get() + '\n')
            if out_queue.empty():
                break
    end = time.time()
    print(end - start)

我加的stop和清理所有變量

    def start_service_from_pause(self, task_id):
        self.service_enabled = True

    def pause_service(self, task_id):

        self.service_enabled = False

    def stop(self, task_id):
        self.is_stoped = False

    def _check_stop(self,task_id):
        """檢查線程池中所有線程是否全部運行完"""
        finish_num = 0
        for thread in self.li:
            if not thread.isAlive():
                finish_num += 1

        return False if finish_num == len(self.li) else True

    def stop_all(self, task_id):
        """掉用線程體stop方法，停止所有線程"""
        for thread in self.li:
            if not thread.isAlive():
                print('thread die:',thread)
            else:
                print('thread alive:', thread)
                thread.stop()


    def __clear_env(self,task_id,locals,globals):
        """清理所有變量"""
        for key in locals:
            del key
            gc.collect()
        for key in globals:
            del key
            gc.collect()

    def __find_all_varibles(self,task_id,locals,globals):
        # print(locals)
        for key in locals:
            print('所有局部變量名*****************************',key)
        for key in globals:
            print('所有全局變量名*****************************',key)