一、選擇python RPC framework
QAM http://packages.python.org/qam/introduction.html
基於carrot消息框架（AMQP協議） http://ask.github.com/carrot/introduction.htmlQAM目前已經不再被積極維護了，它的替代品是callme，carrot也被kombu取代
callme http://pypi.python.org/pypi/callmeveasy_intall callme #今天運氣不好，經常出現502網關錯誤，所以經常要手動下載安裝
PS: 對於callme框架，rpc server如果使用多線程模式的話併發會比較好，但是需要考慮多個rpc調用併發訪問同一個資源的問題
[xudongsong@vh212 tmp]$ cat callme_server.py
import callme, time
g_count = 0
def count():
global g_count
g_count += 1
return g_count
server = callme.Server(server_id='fooserver_1', amqp_host ='localhost', threaded = True)
server.register_function(count, 'count')
server.start()
[xudongsong@vh212 tmp]$ cat callme_client.py
import callme
import logging
import threading
logging.basicConfig(level=logging.INFO, format="%(threadName)s %(asctime)s %(levelname)s [%(filename)s:%(lineno)d]%(message)s")
def thread_body():
proxy = callme.Proxy(amqp_host ='localhost')
while True:
logging.info(proxy.use_server('fooserver_1').count())
if __name__ == '__main__':
threadList = list()
for i in range(10):
th = threading.Thread(target = thread_body)
th.daemon = True
th.start()
threadList.append(th)
for th in threadList:
th.join()經測試，rpc server設置threaded = True的時候rpc client會收到重複的一些數據； rpc server設置threaded = False的時候不會有這個問題。但是server端不使用多線程的話併發會比較差，所以可以如下改進：
[xudongsong@vh212 tmp]$ cat callme_server.py
import callme, time, threading
lock = threading.Lock()
g_count = 0
def count():
lock.acquire()
global g_count
g_count += 1
lock.release()
return g_count
server = callme.Server(server_id='fooserver_1', amqp_host ='localhost', threaded = True)
server.register_function(count, 'count')
server.start()二、RabbitMQ
而這一切的一切都需要AMQP服務端的存在，我選擇是的RabbitMQ
RabbitMQ 官網 http://www.rabbitmq.com/wiki http://en.wikipedia.org/wiki/RabbitMQ
(Erlang真是很強大呢：The RabbitMQ server is written in Erlang and is built on the Open Telecom Platform framework for clustering and failover)
RabbitMQ安裝過程：
yum install rabbitmq-server（用yum list rabbitmq-server可以看到我安裝的版本是2.6.1）sudo /etc/init.d/rabbitmq-server start
或者這樣啓動：rabbitmq-server -detachedrabbitmqctl --help
rabbitmqctl status沒有插件管理工具rabbitmq-plugins？http://stackoverflow.com/questions/8548983/how-to-install-rabbitmq-management-plugin-rabbitmq-plugins
版本太低，插件都要手動去官網下載，太不方便了，改用官網釋放的最新版本吧
http://www.rabbitmq.com/install-generic-unix.html 下載解壓縮即可
整體配置：cat /home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmq-defaults
環境配置是CONF_ENV_FILE所指示的文件（http://www.rabbitmq.com/configure.html#customise-general-unix-environment）組件配置是CONFIG_FILE所指示的文件（http://www.rabbitmq.com/configure.html#configuration-file）
啓動：[dongsong@localhost sbin]$ /home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmq-server -detached
./rabbitmq-server: line 85: erl: command not found
安裝Erlang:http://www.erlang.org/download.html 下載 解壓 ./configure --prefix=xx; make; make install
停止：/home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmqctl stop
狀態：/home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmqctl status
啓用管理插件： /home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmq-plugins enable rabbitmq_management
http://server-name:15672/
三、昨天（2013.1.29）看multiprocessing模塊發現managers用來做rpc框架更方便
[dongsong@localhost python_study]$ cat process_model_v2.py
#encoding=utf-8
import multiprocessing, time, Queue, sys, random
from multiprocessing import Process
from multiprocessing.managers import BaseManager
HOST = '127.0.0.1'
PORT = 50000
AUTH_KEY = 'a secret'
class QueueManager(BaseManager): pass
class QueueProc(Process):
def __init__(self, *args, **kwargs):
self.queueObj = Queue.Queue()
super(QueueProc, self).__init__(*args, **kwargs)
def run(self):
QueueManager.register('get_queue', callable = lambda:self.queueObj)
manager = QueueManager(address = (HOST,PORT), authkey = AUTH_KEY)
server = manager.get_server()
print '%s(%s) started....' % (self.name, self.pid)
server.serve_forever()
print '%s(%s) exit' % (self.name, self.pid)
class Worker(Process):
def run(self):
QueueManager.register('get_queue')
manager = QueueManager(address = (HOST,PORT), authkey = AUTH_KEY)
manager.connect()
self.queueObj = manager.get_queue()
while True:
task = self.queueObj.get()
print '%s(%s) get task "%s", %s left in queue' % (self.name, self.pid, task, self.queueObj.qsize())
time.sleep(random.randrange(5))
class Scheduler(Process):
def run(self):
QueueManager.register('get_queue')
manager = QueueManager(address = (HOST,PORT), authkey = AUTH_KEY)
manager.connect()
self.queueObj = manager.get_queue()
taskId = 0
while True:
task = 'task-%d' % taskId
taskId += 1
self.queueObj.put(task)
print '%s(%s) put task "%s", %s left in queue' % (self.name, self.pid, task, self.queueObj.qsize())
time.sleep(random.randrange(5))
if __name__ == '__main__':
queueProc = QueueProc()
print 'queueProc deamon = %s; is_alive() = %s' % (queueProc.daemon, queueProc.is_alive())
queueProc.daemon = True #父進程退出時queueProc被terminate掉，queueProc不允許創建子進程
queueProc.start()
print 'queueProc deamon = %s; is_alive() = %s' % (queueProc.daemon, queueProc.is_alive())
while not queueProc.is_alive():
print 'queueProc.is_alive() = %s' % queueProc.is_alive()
scheduler = Scheduler()
scheduler.daemon = True
scheduler.start()
workerList = [Worker() for i in range(1)]
for worker in workerList:
worker.daemon = True
worker.start()
currentProc = multiprocessing.current_process()
print '%s(%s) is the master...' % (currentProc.name, currentProc.pid)
queueProc.join()
scheduler.join()
worker.join()
[dongsong@localhost python_study]$ vpython process_model_v2.py
queueProc deamon = False; is_alive() = False
queueProc deamon = True; is_alive() = True
MainProcess(3341) is the master...
QueueProc-1(3342) started....
Scheduler-2(3343) put task "task-0", 1 left in queue
Worker-3(3344) get task "task-0", 1 left in queue
Scheduler-2(3343) put task "task-1", 1 left in queue
Scheduler-2(3343) put task "task-2", 2 left in queue
Worker-3(3344) get task "task-1", 2 left in queue
Scheduler-2(3343) put task "task-3", 2 left in queue監控進程消耗的內存
[dongsong@localhost python_study]$ cat monitor_memory.py
#encoding=utf-8
import multiprocessing, time
import os
_proc_status = '/proc/%d/status' % os.getpid()
_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
'KB': 1024.0, 'MB': 1024.0*1024.0}
def _VmB(VmKey):
'''Private.
'''
global _proc_status, _scale
# get pseudo file /proc/<pid>/status
try:
t = open(_proc_status)
v = t.read()
t.close()
except:
return 0.0 # non-Linux?
# get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
i = v.index(VmKey)
v = v[i:].split(None, 3) # whitespace
if len(v) < 3:
return 0.0 # invalid format?
# convert Vm value to bytes
return float(v[1]) * _scale[v[2]]
def memory(since=0.0):
'''Return memory usage in bytes.
'''
return _VmB('VmSize:') - since
def resident(since=0.0):
'''Return resident memory usage in bytes.
'''
return _VmB('VmRSS:') - since
def stacksize(since=0.0):
'''Return stack size in bytes.
'''
return _VmB('VmStk:') - since
def FetchMemSize(pid = None):
if pid == None:
proc = multiprocessing.current_process()
pid = proc.pid
print 'current process pid is %s' % pid
statusInfos = file('/proc/%s/status' % pid,'r').read()
indexNum = statusInfos.index('VmRSS:')
print '\t'.join(statusInfos[indexNum:].split(None, 3)[0:3])
if __name__ == '__main__':
d = dict()
dIndex = 0
while True:
d[dIndex] = 'hello'*10000
FetchMemSize()
print resident()
time.sleep(3)
dIndex += 1
[dongsong@localhost python_study]$ vpython monitor_memory.py
current process pid is 3667
VmRSS: 4800 kB
4919296.0
current process pid is 3667
VmRSS: 4876 kB
4993024.0
current process pid is 3667
VmRSS: 4924 kB
5042176.0
current process pid is 3667
VmRSS: 4972 kB
5091328.0