python--文本處理(二)

接上一篇：http://zhengbin.blog.51cto.com/2989505/1281577

更改下思路，先將日誌分析到字典裏，再從字典裏取數據打印：

返回字典的代碼

#!/usr/bin/python
#coding=utf-8
import os,re
import time,datetime,sys
def readLog(logpath,logSeekFile,maxReadCount=5000):
    canshu=logpath
    rizhi_dx=maxReadCount
    rizhi_open=file(canshu,'r')
    temp_wenjian=logSeekFile
    if os.path.isfile(temp_wenjian):
        pass
    else:
        temp_1=file(temp_wenjian,'w')
        temp_1.write('0')
        temp_1.close()
    temp=file(temp_wenjian,'r')
    rizhi_open.seek(int(temp.readline()))
    temp.close()
    dictionary={}
    rizhi_size=0
    last_time = 0
    readbyte=rizhi_open.tell()
    now=int(time.time())
    while True:
        rizhi_1 = rizhi_open.readline(1024)
        rizhi_size=rizhi_size+1
        if not rizhi_1:
           break
        log_array=rizhi_1.strip('\n').split(',')
        if len(log_array)==5: 
            log_time=int(time.mktime(datetime.datetime.strptime(log_array[0],'%Y%m%d%H%M%S').timetuple()))
            log_metris=log_array[1]
            log_count=log_array[2]
            log_value=log_array[3]
            log_tag=log_array[4]
            if log_time > now:
                break
            if rizhi_size==rizhi_dx:
                last_time=log_time
            if log_time!=last_time and  last_time>0:
                break
            if not dictionary.has_key(log_time):
                dictionary[log_time]={}
            if not dictionary[log_time].has_key(log_metris):
                dictionary[log_time][log_metris]={}
                metrises =dictionary[log_time][log_metris]
                metrises["count"]=0
                metrises["totalValue"]=0
                metrises["error"]=0
                metrises["min"]=0
                metrises["max"]=0
            metrises =dictionary[log_time][log_metris]
            if metrises["min"]==0 or  metrises["min"]>log_value:
                metrises["min"] = float(log_value)
            if metrises["max"]==0 or  metrises["max"]<log_value:
                metrises["max"] = float(log_value)
            metrises["count"] += int(log_count)
            metrises["totalValue"]+=float(log_value)
            if log_tag <=0:
                metrises["error"]+=1
            readbyte=rizhi_open.tell()
    temp1=open(temp_wenjian,'w')
    temp1.write(str(readbyte))
    temp1.close()
    return dictionary
if __name__=="__main__":
    while 1:
        abcd=readLog('/home/test/tsdb-0910.txt','/home/test/rizhi/temp',10)
        print abcd
        time.sleep(3)

打印的python腳本：

#!/usr/bin/python
#coding=utf-8
import sys,time
log_analysis_list='/home/test/tcollector-master/collectors/lib'
def opentsdb(logpath,logSeekFile,appname,maxReadCount=100):
    sys.path.append(log_analysis_list)
    from readlog import readLog
    rizhi=readLog(logpath,logSeekFile,maxReadCount)
    for time_key,time_value in sorted(rizhi.items()):
       for metris_key,metris_value in time_value.items():
           keyID=keyID.replace('(','').replace(')','')
           if ':' in metris_key:
               keyID=metris_key.split(':')[0]
               masterID=metris_key.split(':')[1]
               print keyID,time_key,metris_value['count'],"type=count master=%s appname=%s" %(masterID,appname)
               print keyID,time_key,metris_value['max'],"type=max master=%s appname=%s" %(masterID,appname)
               print keyID,time_key,metris_value['min'],"type=min master=%s appname=%s" %(masterID,appname)
               print keyID,time_key,metris_value['error'],"type=error master=%s appname=%s" %(masterID,appname)
               print keyID,time_key,metris_value['totalValue'],"type=totalValue master=%s appname=%s" %(masterID,appname)
               print keyID,time_key,metris_value['totalValue']/metris_value['count'],"type=avg master=%s appname=%s" %(masterID,appname)
           else:
               print metris_key,time_key,metris_value['count'],"type=count appname=%s" %(appname)
               print metris_key,time_key,metris_value['max'],"type=max appname=%s" %(appname)
               print metris_key,time_key,metris_value['min'],"type=min appname=%s" %(appname)
               print metris_key,time_key,metris_value['error'],"type=error appname=%s" %(appname)
               print metris_key,time_key,metris_value['totalValue'],"type=totalValue appname=%s" %(appname)
               print metris_key,time_key,metris_value['totalValue']/metris_value['count'],"type=avg appname=%s" %(appname)
if __name__=="__main__":
    while True:
        opentsdb('/home/test/esInfinispanTaskService.log','/mnt/temp1','A1')
        sys.stdout.flush()
        time.sleep(2)