一、选择python RPC framework
QAM http://packages.python.org/qam/introduction.html
基于carrot消息框架（AMQP协议） http://ask.github.com/carrot/introduction.htmlQAM目前已经不再被积极维护了，它的替代品是callme，carrot也被kombu取代
callme http://pypi.python.org/pypi/callmeveasy_intall callme #今天运气不好，经常出现502网关错误，所以经常要手动下载安装
PS: 对于callme框架，rpc server如果使用多线程模式的话并发会比较好，但是需要考虑多个rpc调用并发访问同一个资源的问题
[xudongsong@vh212 tmp]$ cat callme_server.py
import callme, time
g_count = 0
def count():
global g_count
g_count += 1
return g_count
server = callme.Server(server_id='fooserver_1', amqp_host ='localhost', threaded = True)
server.register_function(count, 'count')
server.start()
[xudongsong@vh212 tmp]$ cat callme_client.py
import callme
import logging
import threading
logging.basicConfig(level=logging.INFO, format="%(threadName)s %(asctime)s %(levelname)s [%(filename)s:%(lineno)d]%(message)s")
def thread_body():
proxy = callme.Proxy(amqp_host ='localhost')
while True:
logging.info(proxy.use_server('fooserver_1').count())
if __name__ == '__main__':
threadList = list()
for i in range(10):
th = threading.Thread(target = thread_body)
th.daemon = True
th.start()
threadList.append(th)
for th in threadList:
th.join()经测试，rpc server设置threaded = True的时候rpc client会收到重复的一些数据； rpc server设置threaded = False的时候不会有这个问题。但是server端不使用多线程的话并发会比较差，所以可以如下改进：
[xudongsong@vh212 tmp]$ cat callme_server.py
import callme, time, threading
lock = threading.Lock()
g_count = 0
def count():
lock.acquire()
global g_count
g_count += 1
lock.release()
return g_count
server = callme.Server(server_id='fooserver_1', amqp_host ='localhost', threaded = True)
server.register_function(count, 'count')
server.start()二、RabbitMQ
而这一切的一切都需要AMQP服务端的存在，我选择是的RabbitMQ
RabbitMQ 官网 http://www.rabbitmq.com/wiki http://en.wikipedia.org/wiki/RabbitMQ
(Erlang真是很强大呢：The RabbitMQ server is written in Erlang and is built on the Open Telecom Platform framework for clustering and failover)
RabbitMQ安装过程：
yum install rabbitmq-server（用yum list rabbitmq-server可以看到我安装的版本是2.6.1）sudo /etc/init.d/rabbitmq-server start
或者这样启动：rabbitmq-server -detachedrabbitmqctl --help
rabbitmqctl status没有插件管理工具rabbitmq-plugins？http://stackoverflow.com/questions/8548983/how-to-install-rabbitmq-management-plugin-rabbitmq-plugins
版本太低，插件都要手动去官网下载，太不方便了，改用官网释放的最新版本吧
http://www.rabbitmq.com/install-generic-unix.html 下载解压缩即可
整体配置：cat /home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmq-defaults
环境配置是CONF_ENV_FILE所指示的文件（http://www.rabbitmq.com/configure.html#customise-general-unix-environment）组件配置是CONFIG_FILE所指示的文件（http://www.rabbitmq.com/configure.html#configuration-file）
启动：[dongsong@localhost sbin]$ /home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmq-server -detached
./rabbitmq-server: line 85: erl: command not found
安装Erlang:http://www.erlang.org/download.html 下载 解压 ./configure --prefix=xx; make; make install
停止：/home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmqctl stop
状态：/home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmqctl status
启用管理插件： /home/dongsong/rabbitmq_server-3.0.0/sbin/rabbitmq-plugins enable rabbitmq_management
http://server-name:15672/
三、昨天（2013.1.29）看multiprocessing模块发现managers用来做rpc框架更方便
[dongsong@localhost python_study]$ cat process_model_v2.py
#encoding=utf-8
import multiprocessing, time, Queue, sys, random
from multiprocessing import Process
from multiprocessing.managers import BaseManager
HOST = '127.0.0.1'
PORT = 50000
AUTH_KEY = 'a secret'
class QueueManager(BaseManager): pass
class QueueProc(Process):
def __init__(self, *args, **kwargs):
self.queueObj = Queue.Queue()
super(QueueProc, self).__init__(*args, **kwargs)
def run(self):
QueueManager.register('get_queue', callable = lambda:self.queueObj)
manager = QueueManager(address = (HOST,PORT), authkey = AUTH_KEY)
server = manager.get_server()
print '%s(%s) started....' % (self.name, self.pid)
server.serve_forever()
print '%s(%s) exit' % (self.name, self.pid)
class Worker(Process):
def run(self):
QueueManager.register('get_queue')
manager = QueueManager(address = (HOST,PORT), authkey = AUTH_KEY)
manager.connect()
self.queueObj = manager.get_queue()
while True:
task = self.queueObj.get()
print '%s(%s) get task "%s", %s left in queue' % (self.name, self.pid, task, self.queueObj.qsize())
time.sleep(random.randrange(5))
class Scheduler(Process):
def run(self):
QueueManager.register('get_queue')
manager = QueueManager(address = (HOST,PORT), authkey = AUTH_KEY)
manager.connect()
self.queueObj = manager.get_queue()
taskId = 0
while True:
task = 'task-%d' % taskId
taskId += 1
self.queueObj.put(task)
print '%s(%s) put task "%s", %s left in queue' % (self.name, self.pid, task, self.queueObj.qsize())
time.sleep(random.randrange(5))
if __name__ == '__main__':
queueProc = QueueProc()
print 'queueProc deamon = %s; is_alive() = %s' % (queueProc.daemon, queueProc.is_alive())
queueProc.daemon = True #父进程退出时queueProc被terminate掉，queueProc不允许创建子进程
queueProc.start()
print 'queueProc deamon = %s; is_alive() = %s' % (queueProc.daemon, queueProc.is_alive())
while not queueProc.is_alive():
print 'queueProc.is_alive() = %s' % queueProc.is_alive()
scheduler = Scheduler()
scheduler.daemon = True
scheduler.start()
workerList = [Worker() for i in range(1)]
for worker in workerList:
worker.daemon = True
worker.start()
currentProc = multiprocessing.current_process()
print '%s(%s) is the master...' % (currentProc.name, currentProc.pid)
queueProc.join()
scheduler.join()
worker.join()
[dongsong@localhost python_study]$ vpython process_model_v2.py
queueProc deamon = False; is_alive() = False
queueProc deamon = True; is_alive() = True
MainProcess(3341) is the master...
QueueProc-1(3342) started....
Scheduler-2(3343) put task "task-0", 1 left in queue
Worker-3(3344) get task "task-0", 1 left in queue
Scheduler-2(3343) put task "task-1", 1 left in queue
Scheduler-2(3343) put task "task-2", 2 left in queue
Worker-3(3344) get task "task-1", 2 left in queue
Scheduler-2(3343) put task "task-3", 2 left in queue监控进程消耗的内存
[dongsong@localhost python_study]$ cat monitor_memory.py
#encoding=utf-8
import multiprocessing, time
import os
_proc_status = '/proc/%d/status' % os.getpid()
_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
'KB': 1024.0, 'MB': 1024.0*1024.0}
def _VmB(VmKey):
'''Private.
'''
global _proc_status, _scale
# get pseudo file /proc/<pid>/status
try:
t = open(_proc_status)
v = t.read()
t.close()
except:
return 0.0 # non-Linux?
# get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
i = v.index(VmKey)
v = v[i:].split(None, 3) # whitespace
if len(v) < 3:
return 0.0 # invalid format?
# convert Vm value to bytes
return float(v[1]) * _scale[v[2]]
def memory(since=0.0):
'''Return memory usage in bytes.
'''
return _VmB('VmSize:') - since
def resident(since=0.0):
'''Return resident memory usage in bytes.
'''
return _VmB('VmRSS:') - since
def stacksize(since=0.0):
'''Return stack size in bytes.
'''
return _VmB('VmStk:') - since
def FetchMemSize(pid = None):
if pid == None:
proc = multiprocessing.current_process()
pid = proc.pid
print 'current process pid is %s' % pid
statusInfos = file('/proc/%s/status' % pid,'r').read()
indexNum = statusInfos.index('VmRSS:')
print '\t'.join(statusInfos[indexNum:].split(None, 3)[0:3])
if __name__ == '__main__':
d = dict()
dIndex = 0
while True:
d[dIndex] = 'hello'*10000
FetchMemSize()
print resident()
time.sleep(3)
dIndex += 1
[dongsong@localhost python_study]$ vpython monitor_memory.py
current process pid is 3667
VmRSS: 4800 kB
4919296.0
current process pid is 3667
VmRSS: 4876 kB
4993024.0
current process pid is 3667
VmRSS: 4924 kB
5042176.0
current process pid is 3667
VmRSS: 4972 kB
5091328.0