Python3多進程

1. multiprocessing模塊

1.1 直接調用

import multiprocessing
import time
def f(name):
    time.sleep(1)
    print("hello", name, time.ctime())

if __name__ == "__main__":
    p_list = []
    for i in range(3):
        p = multiprocessing.Process(target=f, args=("alvin",))
        p_list.append(p)
        p.start()
    for p in p_list:
        p.join()
    print("end", time.ctime())

1.2 繼承式調用

import multiprocessing
import time
class MyProcess(multiprocessing.Process):
    def __init__(self, name):
        super(MyProcess, self).__init__()
        self.name = name

    def run(self):
        time.sleep(1)
        print("hello", self.name, time.ctime())

if __name__ == '__main__':
    p_list = []
    for i in range(3):
        p = MyProcess("alvin")
        p.start()
        p_list.append(p)
    for p in p_list:
        p.join()
    print("end", time.ctime())

1.3 父進程和子進程的關係

import multiprocessing
import os
import time

def info(title):
    print(title)
    print("module name:", __name__)
    print("parent process:", os.getppid())   #父進程的ID
    print("process id:", os.getpid())        #當前進程的ID

def f(name):
    info('\033[31;1mfunction f\033[0m')
    print("hello", name)

if __name__ == '__main__':
    info('\033[32;1mmain process line\033[0m')
    time.sleep(3)
    p = multiprocessing.Process(target=f, args=('bob',))
    p.start()
    p.join()

2. 進程間通訊

不同進程間內存是不共享的，要實現兩個進程間的數據交換，可使用Queue和Pipe，使用Managers來進行數據共享。

2.1 Queue隊列

通過pickle序列化存儲起來，傳遞給另一個進程，然後另一個進程反pickle取出來

from multiprocessing import Queue,Process
def f(q):
    q.put([42, 2, 'hello'])
    print("\033[36;1msub q id\033[0m", id(q))

if __name__ == '__main__':
    q = Queue()
    p_list = []
    print("\033[31;1mmain q id\033[0m",id(q))
    for i in range(3):
        p = Process(target=f, args=(q,))
        p_list.append(p)
        p.start()
    print(q.get())
    print(q.get())
    print(q.get())
    for p in p_list:
        p.join()

在Linux上四個ID全部相同

2.2 Pipe管道

from multiprocessing import Pipe,Process
import os
def f(con):
    con.send([42, None, 'hello'])
    print(con.recv(), "int the %s" % os.getpid())
    con.close()

if __name__ == '__main__':
    parent_con, child_con = Pipe()
    p = Process(target=f, args=(child_con,))
    p2 = Process(target=f, args=(child_con,))
    p.start()
    p2.start()
    print(parent_con.recv())
    print(parent_con.recv())
    parent_con.send('hello')
    parent_con.send('python')
    p.join()

2.3 Manager數據共享

from multiprocessing import Process, Manager
def f(d, l, n):
    d[n] = '1'
    d['2'] = 2
    d[0.25] = None
    l.append(n)
    print("sub:",id(d))

if __name__ == '__main__':
    with Manager() as manager:
        d = manager.dict()        #創建一個字典
        l = manager.list(range(5))    #創建一個列表
        p_list = []
        print("main:",id(d))
        for i in range(10):           #創建10個進程
            p = Process(target=f, args=(d, l, i))
            p.start()
            p_list.append(p)
        for res in p_list:
            res.join()
        print(d)
        print(l)