本項目幾個要點:
1.數據庫中有中文,且爲unicode編碼,本項目環境爲python2
2.數據庫中時間字段爲13位的字符型的時間戳,導出時需要改成正常的時間
3.導數需求爲每一個小時導前一個小時的日誌
4.導出後,涉及到將文件從此服務器轉移到另一臺服務器上(都是linux環境)
5.涉及到帶密碼的scp傳輸文件和python中執行shell命令
# -*- coding: utf-8 -*-
import MySQLdb as mdb
import codecs
import datetime
import time
import os
log_path='/data/log/sqllog_'
#log_file爲日誌文件,start_time爲日誌起始時間,end_time爲日誌結束時間
def get_start_time():
dateFormat = "%Y-%m-%d %H:00:00"
end_time = datetime.datetime.now().strftime(dateFormat)
log_time = datetime.datetime.now().strftime("%Y%m%d%H")
timeArray=time.strptime(end_time, "%Y-%m-%d %H:%M:%S")
end_time=int(time.mktime(timeArray))*1000
start_time=end_time-3600000
log_file=log_path+log_time
return log_file,start_time,end_time
def export_data(log_file,start_time,end_time):
conn = mdb.connect(
host = '192.168.1.11',
user = 'username',
passwd = 'passwd',
db = 'dbname',
port = 3306,
charset = 'utf8'
)
cur = conn.cursor(mdb.cursors.DictCursor)
sql = "select uid,pid,menuid,sid,pt,FROM_UNIXTIME(CONVERT(st/1000,signed),'%Y-%m-%d %T') as stime,FROM_UNIXTIME(CONVERT(et/1000,signed),'%Y-%m-%d %T') as etime from log_info where st>='"+str(start_time)+"' and st<'"+str(end_time)+"';"
cur.execute(sql)
rows=cur.fetchall()
with codecs.open(log_file,'w','utf-8') as fh:
for row in rows:
fh.write("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" %(row["uid"],row["pid"],row["menuid"],row["sid"],row["pt"],row["stime"],row["etime"]))
fh.write("\n")
cur.close()
conn.close()
#將文件轉移到另一臺服務器
def scp_file(log_file):
cmd='sshpass -p passwd2 scp -P 22 '+ log_file+' [email protected]:/data/log'
os.system(cmd)
if __name__=='__main__':
log_file,start_time,end_time=get_start_time()
export_data(log_file,start_time,end_time)
scp_file(log_file)
print("ok")