Redis BIgKey自動分析處理

    使用開源rdb工具與Lepus進行REDIS RDB的數據分析，相信使用REDIS，對CACHE的重要性不言而喻，儘可做到事前-預判，堅決不做背鍋俠........

    生成環境需要分析的RDB會很多，如一個FE展示着多個DASHBOARD，一個DASHBOARD下有一個或多個GROUP，監控的REDIS主要從Lepus獲取(可根據實際是否使用)，如何避免手動逐個RDB進行分析，進行重複操作....................

    實現過程:

         1、結合LEPUS 查找存在的REDIS

             1、主機信息：存儲ROOT密碼用來PARAMIKO登陸

             2、LEPUS定義監控信息

         2、從FE獲取記錄需要分析的RDB

         3、執行分析郵件告警

      ###實踐過程需要解決python一些依賴包:MySQL Redis RDB  

pip install python-lzf

定義表:

 int(11) NOT NULL AUTO_INCREMENT,
  `idc` tinyint(1) DEFAULT '1',
  `productName` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '業務線名稱',
  `ip` varchar(16) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'REDIS地址',
  `port` int(11) DEFAULT NULL COMMENT 'REDIS端口',
  `bgsave` tinyint(1) DEFAULT '0' COMMENT '執行狀態',
  `rdb_last_save_time_begin` int(11) DEFAULT '1461645350' COMMENT '第一次執行前時間',
  `rdb_last_save_time_over` int(11) DEFAULT '1461645350' COMMENT '執行BGSAVE後時間',
  `rdb_last_bgsave_status` varchar(10) COLLATE utf8mb4_unicode_ci DEFAULT 'ok',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT "業務線定義分析的RDB"

 

CREATE TABLE `mysql_ssh_pass` (
  `id` int(32) NOT NULL AUTO_INCREMENT,
  `idc` int(16) DEFAULT NULL COMMENT '所述IDC編號',
  `ip` varchar(15) CHARACTER SET utf8 DEFAULT NULL COMMENT '數據庫ip',
  `os` varchar(30) CHARACTER SET utf8 DEFAULT NULL COMMENT '服務器系統版本',
  `username` varchar(20) CHARACTER SET utf8 DEFAULT NULL COMMENT 'ssh用戶名',
  `password` varchar(128) CHARACTER SET utf8 DEFAULT NULL COMMENT 'ssh密碼',
  `comment` varchar(500) CHARACTER SET utf8 NOT NULL DEFAULT '' COMMENT '備註',
  PRIMARY KEY (`id`),
  UNIQUE KEY `uniqa` (`ip`)
) ENGINE=InnoDB AUTO_INCREMENT=307 DEFAULT CHARSET=utf8mb4 COMMENT "定義系統登陸信息"

###可有可無，主要使用LEPUS 可以直接使用

CREATE TABLE `db_servers_redis` (
  `id` int(10) NOT NULL AUTO_INCREMENT,
  `host` varchar(30) COLLATE utf8_unicode_ci NOT NULL,
  `port` varchar(10) COLLATE utf8_unicode_ci NOT NULL,
  `password` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
  `tags` varchar(50) COLLATE utf8_unicode_ci DEFAULT NULL,
  `monitor` tinyint(2) DEFAULT '1',
  `send_mail` tinyint(2) DEFAULT '1',
  `send_mail_to_list` varchar(2000) COLLATE utf8_unicode_ci DEFAULT NULL,
  `send_wx` tinyint(4) DEFAULT '1',
  `send_wx_to_list` tinyint(255) DEFAULT NULL,
  `send_sms` tinyint(2) DEFAULT '0',
  `send_sms_to_list` varchar(255) COLLATE utf8_unicode_ci DEFAULT NULL,
  `alarm_connected_clients` int(2) NOT NULL DEFAULT '1',
  `alarm_command_processed` int(2) NOT NULL DEFAULT '1',
  `alarm_blocked_clients` int(2) NOT NULL DEFAULT '1',
  `threshold_warning_connected_clients` int(4) NOT NULL DEFAULT '1000',
  `threshold_warning_command_processed` int(4) NOT NULL DEFAULT '12000',
  `threshold_warning_blocked_clients` int(4) NOT NULL DEFAULT '50',
  `threshold_critical_connected_clients` int(4) NOT NULL DEFAULT '3000',
  `threshold_critical_command_processed` int(4) NOT NULL DEFAULT '15000',
  `threshold_critical_blocked_clients` int(4) NOT NULL DEFAULT '150',
  `is_delete` tinyint(1) NOT NULL DEFAULT '0',
  `display_order` smallint(4) NOT NULL DEFAULT '0',
  `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
  PRIMARY KEY (`id`),
  KEY `idx_host` (`host`)
) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci

###郵件模板SendMail.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import string
import time
import datetime
import MySQLdb
import logging
import logging.config
from multiprocessing import Process;
import subprocess
import threading
import smtplib
from email.mime.text import MIMEText
#from html import HTML
from email.utils import formataddr
reload(sys) 
sys.setdefaultencoding('utf-8')
def sendSmail(subject,msg,fromemail,emailpasswd,toemail):
	user=fromemail
        pwd=emailpasswd
        to=toemail
        nowtime=time.strftime('%Y-%m-%d %H:%M:%S')
        msg=MIMEText(msg,'html','utf-8')
        if not isinstance(subject,unicode):
            subject = unicode(subject)
        msg["Accept-Language"]="zh-CN"
        msg["Accept-Charset"]="ISO-8859-1,utf-8"
        msg["Subject"]=subject
        msg["From"]=user
        msg["To"]=",".join(to)
        try:
                s=smtplib.SMTP('mail.hualala.com',587)
                s.login(user,pwd)
                s.sendmail(fromemail,to,msg.as_string())
                s.quit()
                print "[%s]INFO:Email send Success!"% nowtime
        except smtplib.SMTPException,e:
                print "[%s]ERROR:Email send Faild,%s"%(nowtime,e)

##告警模板alarm_sendmail.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import string
import time
import datetime
import MySQLdb
import logging
import logging.config
from multiprocessing import Process;
import subprocess
import threading
import smtplib
from email.mime.text import MIMEText
from email.utils import formataddr
import SendMail as sendMail
reload(sys) 
sys.setdefaultencoding('utf-8')
def sendSlogLog(host,port,data,productName,sendList):
	bb=""
	userlist1='''
        <table border="1";cellspacing="0";cellpadding="0";text-align:center;border-color:#b6ff00;boder-collapse:collapse;>  <h1>Codis BigKeys Analysis:'''+str(host)+"_"+str(port)+" "+"【"+productName+"】"+" "+'''</h1>
                            <tr>
                                    <td>Database</td>
                                    <td>Type</td>
                                    <td>Key</td>
                                    <td>Size_in_bytes</td>
                                    <td>Encoding</td>
                                    <td>Num_elements</td>
                                    <td>Len_largest_element</td>
                                    <td>Expiry</td>
                            </tr>'''
	with open(data,'r') as f:
	    for line in f.readlines():
                data=line.strip().split(',')
                aa='''<tr>
		     <td>%s</td>
		     <td>%s</td>
		     <td>%s</td>
		     <td>%s</td>
		     <td>%s</td>
		     <td>%s</td>
		     <td>%s</td>
		     <td>%s</td>
                  </tr>'''%(str(data[0]),str(data[1]),str(data[2]),str(data[3]),str(data[4]),str(data[5]),str(data[6]),str(data[7]))
                bb=bb+aa+"\n"
	    if bb:
                cc=userlist1+bb+" </table>"
	        subject="Redis BigKey Analysis Report"
	        fromemail='郵箱地址'
 	        emailpasswd='郵箱密碼'
	        global to_addrs
	        if sendList.strip()=='':
	            sendList="[email protected];[email protected]"
	        else:
	            sendList=sendList
	        toemail=sendList
	        to_addrs=toemail.split(';')
	        sendMail.sendSmail(subject,cc,fromemail,emailpasswd,to_addrs)

##執行模板:rbd-exe.py

#!/usr/bin/evn python
#-*- coding: utf-8 -*-  
import sys,os,commands,subprocess
import paramiko,socket,time
import alarm_sendmail as al
import base64
reload(sys)
sys.setdefaultencoding('utf-8')
import redis,MySQLdb
def ConnRedis(rdb):
    backup=rdb
    host=rdb['ip']
    port=rdb['port']
    pool=redis.ConnectionPool(host=host,port=port,db=0,decode_responses=True)
    r=redis.Redis(connection_pool=pool)
    cur=conn.cursor()
    if backup['CurrentDate'] !=backup['rdb_last_save_time_over'] and backup['bgsave']==0:
        print "爲執行bgsave.......",backup['CurrentDate'],backup['rdb_last_save_time_over']
        try:
	    beginStart=int(time.time())
	    up="update db_manager.tbl_bigdate_record set bgsave=1 where ip="+"'"+str(host)+"'"+"and port="+str(port)
	    print "執行RDB備份中.........",beginStart
	    cur.execute(up)
            conn.commit()
            r.bgsave()
	    time.sleep(30)
	    info=r.info()
	    sdirName=r.config_get()['dir']
	    dumpName=r.config_get()['dbfilename']
	    rdb_back_status=''
	    result=''
	    if info['rdb_last_bgsave_status']=="ok":
                rdb_back_status='ok'
		result="執行RDB備份成功,完成時間:%s....."%info['rdb_last_save_time']
	    else:
		rdb_back_status='err'
		result="執行備份失敗.....Codis:%s:%s"%(host,port)
	    print result
	    endup="update db_manager.tbl_bigdate_record set bgsave=0,rdb_last_save_time_over="+str(info['rdb_last_save_time'])+",rdb_last_bgsave_status="+"'"+str(rdb_back_status)+"'"+",rdb_last_save_time_begin="+str(beginStart)+" where ip="+"'"+str(host)+"'"+"and port="+str(port)
	    cur.execute(endup)
	    conn.commit()
	    return rdb_back_status,sdirName,dumpName
        except Exception,e:
            print e
    elif backup['CurrentDate']==backup['rdb_last_save_time_over'] and backup['bgsave']==0 and backup['rdb_last_bgsave_status']=='err':
        print "RDB Backup 異常..........Codis:%s:%s"%(host,port)
    else:
	rdb_back_status=3
	print "RDB Backup 可以使用上次備份分析......Codis:%s:%s"%(host,port)
	return rdb_back_status,1

def ConnDB(host,port,username,password,idc):
    #backup={}
    conn=MySQLdb.connect(host=host,port=port,user=username,passwd=password,charset='utf8')
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    #sql="select productName,ip,port,rdbconf,bgsave,date(from_unixtime(rdb_last_save_time_begin))as rdb_last_save_time_begin,date(from_unixtime(rdb_last_save_time_over))as rdb_last_save_time_over,rdb_last_bgsave_status,date(now())as CurrentDate from db_manager.tbl_bigdate_record"
    sql="SELECT cmd.productName as productName,cmd.ip as ip,cmd.port as port,cmd.bgsave as bgsave,DATE(FROM_UNIXTIME(cmd.rdb_last_save_time_begin))AS rdb_last_save_time_begin,DATE(FROM_UNIXTIME(cmd.rdb_last_save_time_over))AS rdb_last_save_time_over,cmd.rdb_last_bgsave_status,DATE(NOW())AS CurrentDate,tmp.send_mail_to_list,pwd.password AS upassword FROM db_manager.tbl_bigdate_record cmd LEFT JOIN lepus.db_servers_redis tmp ON cmd.ip=tmp.host AND cmd.port=tmp.port LEFT JOIN dbinfo.mysql_ssh_pass pwd  ON pwd.ip=cmd.ip WHERE tmp.host IS NOT NULL and cmd.idc={idc}".format(idc=str(idc)) 
    cur.execute(sql)
    backup=cur.fetchall()
#    for i in data:
#        for k,v in i.items():
#           backup[k]=v

    return backup,conn
def LoadSoft(logdir,rdb):
    ##下載包保存文件存儲空間檢查...................解壓存儲路徑檢擦
    Rdbcodis=ConnRedis(rdb)
    if not Rdbcodis:
	print "未獲取到最新得rdb文件........"
    elif  Rdbcodis[0]=="err":
	print "RDB異常......"
    elif Rdbcodis[0]==3:
        print "已分析過RDB......."
    else:
	password_tmp=rdb['upassword']
        upassword=base64.decodestring(password_tmp)
        remote_file=Rdbcodis[1]
	ip=rdb['ip']
        productName=rdb['productName']
        local_file_tmp=logdir+"/"+str(productName)
        print remote_file,local_file_tmp
        if not os.path.exists(local_file_tmp):
	    os.system(r'mkdir -p {}'.format(local_file_tmp))
        remote_file=remote_file+"/"+str(Rdbcodis[2])
        local_file=local_file_tmp+"/"+str(Rdbcodis[2])
        ssh=paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(ip,22,'root',upassword)
        a=ssh.exec_command('data')
        stdin,stdout,stdeer=a
        print stdout.read()
        sftp=paramiko.SFTPClient.from_transport(ssh.get_transport())
        time1=time.time()
        sftp=ssh.open_sftp()
        sftp.get(remote_file,local_file)
        gdown=True
    
        print "下載完成%s,Codis:%s:%s"%(str(time.time()-time1),ip,rdb['port'])
	return local_file
def RdbAnalyse(logdir,rdb):
    newdstDir=LoadSoft(logdir,rdb)
    if not newdstDir:
        print "無任何操作........"
    else:
	beginStart=int(time.time())
        print "執行RDB分析.........",beginStart
        dstDir=newdstDir
	sendList=rdb['send_mail_to_list']
	productName=rdb['productName']
	host=rdb['ip']
	port=rdb['port']
        cmd_exe="/usr/local/bin/rdb -c memory -l 100 "+dstDir
        dstDumpDir=os.path.dirname(dstDir)+"/"+productName+".log"
        log_file=open(dstDumpDir,"w")
        stdout_rdb=sys.stdout
	sys.stdout=log_file
	with os.popen(cmd_exe)as c:
	    rdb_log=c.read()
	    print rdb_log
	log_file.close()
	sys.stdout=stdout_rdb
	cmddel="sed -i 1d"+" "+dstDumpDir
	os.system(cmddel)
	cmdr="sed -i '/^$/d'"+" "+dstDumpDir
	os.system(cmdr)
	endStart=int(time.time())
        print "執行RDB分析完成,共計耗時:%s........."%(endStart-beginStart)
	al.sendSlogLog(host,port,dstDumpDir,productName,sendList)
if __name__=="__main__":
    idc=sys.argv[1]
    logdir="/home/rdb-analyse/analyse"
    sqlbackup_tmp=ConnDB('1.1.1.1',123,'tet','abc',idc)
    conn=sqlbackup_tmp[1]
    for rdb in sqlbackup_tmp[0]:
        RdbAnalyse(logdir,rdb)
 #       else:
  #          print "無匹配項"

###最終執行:python rdb-exe.py number