mysql 主備複製停止問題定位以及狀態監控

1 主備複製停止定位

配置好主從複製後，沒有進行主備複製。通過日誌查看，發現從庫在同步的時候跑錯了。
在從庫上查看日誌：

vim /var/log/mysqld.log

提示error信息是函數創建失敗，缺少DETERMINISTIC。並給出了修改建議：

Fix the problem, and restart the slave SQL thread with "SLAVE START". We stopped at log 'mysql-bin.000003' position 25395

查看slave的狀態：

show slave status\G;

返回結果中明確表示，slave sql不再運行。其中的錯誤提示，和log中看到的一致。

2 計劃修復的方案

越過此條log，繼續之後的執行。
在主庫的機器上，查看主庫的log：

show binlog events IN 'mysql-bin.000003' FROM 54911 LIMIT 50;#需要指定哪個文件（上面的log中有），起始position（依舊在上面的log中）

結果：

從log看到，下一條的position是多少。
在從庫中重新定位同步的起始位置，position就是上面找到的信息。

stop slave;
change master to master_log_file='mysql-bin.000003',master_log_pos=47245;
start slave;

查看狀態，確認啓動同步成功：

show slave status\G;

3 線上監控腳本

如果是線上環境，需要監控slave庫的狀態，及時告警。這裏寫了一個python的腳本，很簡陋。
安裝python：我的環境還有py2，所以這裏指定的名字中都帶“3”

yum install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gcc make
wget https://www.python.org/ftp/python/3.6.2/Python-3.6.2.tar.xz
tar -xvJf  Python-3.6.2.tar.xz
./configure prefix=/usr/local/python3
make && make install
ln -s /usr/local/python3/bin/python3 /usr/bin/python3
export PATH=$PATH:/usr/local/python3/bin

安裝setuptools：

wget https://pypi.python.org/packages/source/s/setuptools/setuptools.zip --no-check-certificate
unzip setuptools.zip
cd setuptools
python3 setup.py build
python3 setup.py install

安裝pip：

wget wget https://pypi.python.org/packages/source/p/pip/pip.tar.gz
tar xvf pip.tar.gz
cd pip
python3 setup.py install

注意：各個軟件的版本下載最新的就好
安裝python的依賴包：

pip install PyMySQL
pip install schedule

監控腳本分爲三個文件，一個main以及mysql監控和發送郵件：

發送郵件
'''
Created on 2018-3-21

@author: yunzhong
'''

import smtplib
from email.mime.text import MIMEText
from email.header import Header

class MailSender(object):
    '''
    classdocs
    '''
    mailHost="smtp.exmail.qq.com"
    mailPort=465
    mailProtocol="smtp"
    mailUsername="[email protected]"
    mailPassword="mail"
    receivers = ['[email protected]']

    def __init__(self):
        '''
        Constructor
        '''
    
    def sendEmail(self, headFrom, headTo, subject,content):
        message = MIMEText(content, 'plain', 'utf-8')
        message['From'] = Header(headFrom, 'utf-8')
        message['To'] =  Header(headTo, 'utf-8')
        
        subject = subject
        message['Subject'] = Header(subject, 'utf-8')
        
        
        try:
            smtpObj = smtplib.SMTP_SSL()
            smtpObj.connect(self.mailHost, self.mailPort)
            smtpObj.login(self.mailUsername,self.mailPassword)
            smtpObj.sendmail(self.mailUsername, self.receivers, message.as_string())
            print ("錯誤，發送電子郵件")
        except smtplib.SMTPException as err:
            print ("Error: 不能發送郵件",err)

mysql監控
'''
Created on 2018-3-21

@author: yunzhong
'''
import pymysql

class MysqlMonitor(object):
    '''
    classdocs
    '''
    mysqlHost = 'host.com'
    user = 'user'
    password = 'root+-*/'
    database = ''
   
    def monitor(self):
        db = pymysql.connect(self.mysqlHost,self.user,self.password,self.database )
        cursor = db.cursor()
        cursor.execute("SHOW SLAVE STATUS")
        results = cursor.fetchall()
        print(results)
        slaveIO = 0
        slaveSql = 0
        for result in results:
            print("host:",result[1])
            print("host:",result[5])
            print("host:",result[10])
            print("host:",result[11])
            if result[10].lower() == 'no' :
                slaveIO = 1
            if result[11].lower() == 'no':
                slaveSql = 10
        db.close()
        return slaveIO + slaveSql

main文件
'''
Created on 2018-3-21

@author: yunzhong
'''
import schedule
import time
from mysqlMonitor import MysqlMonitor
from mailSender import MailSender

if __name__ == '__main__':
    pass

def mo():
    print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())),"開始獲取數據庫slave信息")
    monitor = MysqlMonitor()
    mResult = monitor.monitor()
    print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())),"數據庫slave監控信息",mResult)
    if mResult == 1:
        sender = MailSender()
        sender.sendEmail('mysql告警郵件', '管理員', '數據庫同步監控', '主從數據庫運行異常，請覈查。數據庫slave IO異常')
    elif mResult == 10:
        sender = MailSender()
        sender.sendEmail('mysql告警郵件', '管理員', '數據庫同步監控', '主從數據庫運行異常，請覈查。數據庫slave SQL異常')
    elif mResult > 0:
        sender = MailSender()
        sender.sendEmail('mysql告警郵件', '管理員', '數據庫同步監控', '主從數據庫運行異常，請覈查。數據庫slave SQL異常')
    else:
        print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())),"數據庫slave運行正常")

try:
    print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())),"數據庫slave監控程序開始執行")
    schedule.every(1).minutes.do(mo)
except BaseException as err:
    print ("Error: 不能發送郵件",err)
while True:
    try:
        schedule.run_pending()
        time.sleep(1)
    except BaseException as err:
        print ("Error: 不能發送郵件",err)

python腳本啓動命令：-u參數，是爲了print無緩存，直接輸出到console

nohup python3 -u monitor.py > ./monitor.log 2>&1 &