author：skate
time：2013/12/09
lvs+ldirectord+pacemaker+corosync+mysql實現高可用負載均衡(一)
corosync和pacemaker安裝配置高可用
1. 在所有節點上安裝pacemaker和corosync和crmsh
yum方式安裝
# yum install pacemaker* corosync*
檢測是否安裝成功
[root@localhost mysqlinstall]# service corosync status
corosync is stopped
[root@localhost mysqlinstall]# service pacemaker status
pacemakerd is stopped
[root@localhost mysqlinstall]#
或
[root@master mysqlinstall]# ps auxf | grep pacemaker
root 5264 0.0 0.0 103240 832 pts/0 S+ 00:16 0:00 | \_ grep pacemaker
root 10400 0.1 0.1 73488 2844 ? S Nov29 16:42 /usr/libexec/pacemaker/lrmd
189 10402 0.0 1.4 121908 27448 ? S Nov29 8:00 /usr/libexec/pacemaker/pengine
189 3105 0.0 0.5 94412 11132 ? S Dec05 0:49 \_ /usr/libexec/pacemaker/cib
root 3106 0.0 0.2 94508 4060 ? S Dec05 0:33 \_ /usr/libexec/pacemaker/stonithd
189 3108 0.0 0.1 89388 3080 ? S Dec05 0:35 \_ /usr/libexec/pacemaker/attrd
189 3110 0.0 0.6 145460 12328 ? S Dec05 0:33 \_ /usr/libexec/pacemaker/crmd
[root@master mysqlinstall]# ps auxf | grep corosync
root 5422 0.0 0.0 103240 832 pts/0 S+ 00:17 0:00 | \_ grep corosync
root 3099 0.1 0.3 690528 7580 ? Ssl Dec05 9:46 corosync
crmsh工具下載安裝
# wget
http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-6/x86_64/crmsh-1.2.6-5.1.x86_64.rpm
安裝crmsh需要依賴pssh和python-lxml
# rpm -ivh pssh-2.3.1-3.2.x86_64.rpm
# yum install python-lxml
# rpm -ivh crmsh-1.2.6-5.1.x86_64.rpm
檢測crmsh是否安裝成功
[root@localhost mysqlinstall]# crm --version
1.2.6-5.1 (Build 4f66cc1901854a7cb60d483414ce5cf2d749db60)
[root@localhost mysqlinstall]#
2. 所有節點前提準備
1) 各節點之間主機名互相解析
[root@slave92 mysqlinstall]# more /etc/hosts
192.168.213.91 master
192.168.213.92 slave92
2) 各節點之間時間同步
[root@slave92 mysqlinstall]# crontab -e
#ntp
*/10 * * * * /usr/sbin/ntpdate 10.10.10.70 > /dev/null 2>&1
3) 各節點之間ssh互信
[root@slave92 mysqlinstall]# more login_nopass.sh
#!/bin/bash
#for localhost login others db with no passwd
#by: zxg
#time:2012/10/12
for i in $@
do
if [ ! -f /root/.ssh/id_dsa.pub ];then
ssh-keygen -t dsa
fi
scp /root/.ssh/id_dsa.pub root@$i:/tmp/
echo "copied"
ssh -l root root@$i " [ -e /root/.ssh ] || mkdir /root/.ssh"
ssh -l root root@$i "cat /tmp/id_dsa.pub >> /root/.ssh/authorized_keys"
echo "attaphed"
echo "$i is done!"
ssh -l root root@$i "rm -f /tmp/id_dsa.pub"
done
把這個腳本部署到所有的主機上，然後安如下方式執行
[root@slave92 mysqlinstall]# sh login_nopass.sh 192.168.213.91
[root@master mysqlinstall]# sh login_nopass.sh 192.168.213.92
4) 關閉防火牆與SELinux
# service iptables stop
# vi /etc/selinux/config
3. 詳細配置
3.1 corosync配置
root@master mysqlinstall]# ll /etc/corosync/
total 24
-r-------- 1 root root 128 Nov 20 23:54 authkey
-rw-r--r-- 1 root root 545 Nov 21 20:43 corosync.conf
-rw-r--r-- 1 root root 445 May 15 2013 corosync.conf.example
-rw-r--r-- 1 root root 1084 May 15 2013 corosync.conf.example.udpu
drwxr-xr-x 2 root root 4096 May 15 2013 service.d
drwxr-xr-x 2 root root 4096 May 15 2013 uidgid.d
[root@master mysqlinstall]#
可以看到corosync提供一個樣例文件corosync.conf.example
[root@master mysqlinstall]# cp corosync.conf.example corosync.conf
[root@master mysqlinstall]# more /etc/corosync/corosync.conf
# Please read the corosync.conf.5 manual page
compatibility: whitetank //兼容老版本
totem { //集羣節點間通信協議
version: 2 //版本號
secauth: off //是否開啓安全認證功能
threads: 0 //多少線程
interface { //心跳信息
member {
memberaddr: 192.168.213.91 //在線管理節點
}
member {
memberaddr: 192.168.213.92
}
ringnumber: 0
bindnetaddr: 192.168.213.0 //綁定的網絡，本地那個網卡
mcastport: 5405 //端口
ttl: 1 //端口號
}
transport: udpu
}
logging { //日誌
fileline: off
to_logfile: yes //是否記錄日誌文件中
to_syslog: yes //是否記錄進系統日誌
debug: on //是否開啓調試信息
logfile: /var/log/cluster/corosync.log //日誌位置
timestamp: on //是否記錄時間
logger_subsys {
subsys: AMF
debug: off
}
}
(添加以下信息)
service {
ver: 0
name: pacemaker //使用pacemaker
}
用 man corosync.conf 可以查看所有選項的意思
注：corosync生成key文件會默認調用/dev/random隨機數設備，一旦系統中斷的IRQS的隨機數不夠用，將會產生大量的等待時間，因此，爲了節約時間，我們在生成key之前講random替換成urandom，以便節約時間。
[root@master mysqlinstall]# mv /dev/{random,random.bak}
[root@master mysqlinstall]# ln -s /dev/urandom /dev/random
[root@master mysqlinstall]# cd /etc/corosync
[root@master mysqlinstall]# corosync-keygen
查看生成可以文件“authkey”
[root@master mysqlinstall]# ll /etc/corosync/
total 24
-r-------- 1 root root 128 Nov 20 23:54 authkey
-rw-r--r-- 1 root root 545 Nov 21 20:43 corosync.conf
-rw-r--r-- 1 root root 445 May 15 2013 corosync.conf.example
-rw-r--r-- 1 root root 1084 May 15 2013 corosync.conf.example.udpu
drwxr-xr-x 2 root root 4096 May 15 2013 service.d
drwxr-xr-x 2 root root 4096 May 15 2013 uidgid.d
把文件authkey和corosync.conf複製到slave92上（corosync集羣添加新的節點，也是把authkey和corosync.conf複製到新的節點並修改所有節點上的corosync.conf，如果要在線維護，一定要在資源的維護模式下操作）。
corosync到這個就全配置完了，現在啓動看看corosync是否正常
3.2 啓動所有節點的corosync
[root@master mysqlinstall]# service corosync start
Starting Corosync Cluster Engine (corosync): [ OK ]
[root@master mysqlinstall]# service corosync status
corosync (pid 22130) is running...
[root@master mysqlinstall]#
查看corosync啓動信息
A. 查看corosync引擎是否正常啓動
[root@master mysqlinstall]# grep -e "Corosync Cluster Engine" -e "configuration file" /var/log/cluster/corosync.log
Nov 28 02:05:49 corosync [MAIN ] Corosync Cluster Engine ('1.4.1'): started and ready to provide service.
Nov 28 02:05:49 corosync [MAIN ] Successfully read main configuration file '/etc/corosync/corosync.conf'.
Dec 10 00:24:53 corosync [MAIN ] Corosync Cluster Engine exiting with status 0 at main.c:1858.
B. 查看初始化成員節點通知是否正常發出
[root@master mysqlinstall]# grep TOTEM /var/log/cluster/corosync.log | more
Nov 20 23:55:44 corosync [TOTEM ] Initializing transport (UDP/IP Unicast).
Nov 20 23:55:44 corosync [TOTEM ] Initializing transmit/receive security: libtomcrypt SOBER128/SHA1HMAC (mode 0).
Nov 20 23:55:45 corosync [TOTEM ] The network interface [192.168.213.91] is now up.
Nov 20 23:55:45 corosync [TOTEM ] adding new UDPU member {192.168.213.91}
Nov 20 23:55:45 corosync [TOTEM ] adding new UDPU member {192.168.213.92}
Nov 20 23:55:45 corosync [TOTEM ] A processor joined or left the membership and a new membership was formed.
Nov 20 23:56:17 corosync [TOTEM ] A processor joined or left the membership and a new membership was formed.
C. 檢查啓動過程中是否有錯誤產生
[root@master mysqlinstall]# grep ERROR: /var/log/cluster/corosync.log | more
Nov 20 23:55:45 corosync [pcmk ] ERROR: process_ais_conf: You have configured a cluster using the Pacemaker plugin for Corosync. Th
e plugin is not supported in this environment and will be removed very soon.
Nov 20 23:55:45 corosync [pcmk ] ERROR: process_ais_conf: Please see Chapter 8 of 'Clusters from Scratch' (http://www.clusterlabs.
org/doc) for details on using Pacemaker with CMAN
D. 查看pacemaker是否正常啓動
grep pcmk_startup /var/log/cluster/corosync.log
查看集羣狀態
[root@master mysqlinstall]# crm_mon
Attempting connection to the cluster...
Last updated: Tue Dec 10 02:00:24 2013
Last change: Tue Dec 10 01:17:19 2013 via crm_attribute on master
Stack: classic openais (with plugin)
Current DC: slave92 -
partition with quorum
Version: 1.1.10-1.el6_4.4-368c726
3 Nodes configured, 3 expected votes
4 Resources configured
Online: [ master slave92 ]
OFFLINE: [ slave93 ]
vip (ocf::heartbeat:IPaddr): Started master
Master/Slave Set: ms_MySQL [p_mysql]
Masters: [ master ]
Slaves: [ slave92 ]
vip_r (ocf::heartbeat:IPaddr): Started master
顯示本集羣可用節點
[root@master mysqlinstall]# crm_node -l
1540729024 master member
1557506240 slave92 member
上面看到這個集羣管理了4個資源，corosync本身是不管理資源的，要藉助pacemaker纔可以管理資源
[root@master mysqlinstall]# crm
crm(live)# configure //進入配置模式
crm(live)configure# verify //驗證配置文件
crm(live)configure# show //查看配置文件
crm(live)# status //查看資源狀態
crm(live)resource# reprobe //刷新資源狀態
eg：
crm(live)configure# show
node master \
attributes maintenance="off" standby="off"
node slave92 \
attributes standby="off" maintenance="off"
node slave93
primitive p_mysql ocf:heartbeat:mysql \
params config="/etc/my.cnf" pid="/mysql/data/mysql5528/data/master.pid" replication_user="rep" replication_passwd="rep" max_slave_lag="120" evict_outdated_slaves="false" binary="/usr/local/mysql/bin/mysqld_safe" user="root" test_user="root" test_passwd="root"
socket="/mysql/tmp/mysql.sock" datadir="/mysql/data/mysql5528/data" \
op monitor interval="10s" timeout="30s" role="Master" OCF_CHECK_LEVEL="1" \
op monitor interval="30s" timeout="30s" role="Slave" OCF_CHECK_LEVEL="1" \
op start interval="0" timeout="120s" \
op stop interval="0" timeout="120s"
primitive vip ocf:heartbeat:IPaddr \
params ip="192.168.213.95" nic="eth0:1" \
meta target-role="Started" is-managed="true"
primitive vip_r ocf:heartbeat:IPaddr \
params ip="192.168.213.96" nic="eth0:2" \
meta target-role="Started" is-managed="true" resource-stickiness="100"
ms ms_MySQL p_mysql \
meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" globally-unique="false" target-role="Started" is-managed="true"
colocation vip_before_ms_MySQL_M 500: ms_MySQL:Master vip
order vip_mysql_1 Mandatory: ms_MySQL:promote vip
property $id="cib-bootstrap-options" \
dc-version="1.1.10-1.el6_4.4-368c726" \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes="3" \
stonith-enabled="false" \
no-quorum-policy="ignore" \
last-lrm-refresh="1385485315"
property $id="mysql_replication" \
p_mysql_REPL_INFO="master|mysql-bin.000234|107"
crm(live)configure#
如果沒有STONITH設備，就要先關閉stonith-enabled這個屬性 ，如下
crm(live)configure# property stonith-enabled=false
對於只有兩節點的集羣來講。如果要滿足集羣服務本身正常運行的條件，就需要忽略quorum不能滿足時的集羣狀態檢查
crm(live)configure# property no-quorum-policy=ignore
需要修改兩個文件
1.
[root@master mysqlinstall]# vi /etc/init.d/mysqld
....
....
# Set pid file if not given
#
if test -z "$mysqld_pid_file_path"
then
###by zxg modify
mysqld_pid_file_path=$datadir/master.pid
#mysqld_pid_file_path=$datadir/`hostname`.pid
else
case "$mysqld_pid_file_path" in
....
2.
[root@master mysqlinstall]# vi /usr/lib/ocf/resource.d/heartbeat/mysql
....
....
OCF_RESKEY_config_default="/etc/my.cnf"
OCF_RESKEY_datadir_default="/var/lib/mysql"
OCF_RESKEY_user_default="mysql"
OCF_RESKEY_group_default="mysql"
OCF_RESKEY_log_default="/var/log/mysqld.log"
###by zxg modify
#OCF_RESKEY_pid_default="/var/run/mysql/mysqld.pid"
OCF_RESKEY_pid_default="/mysql/data/mysql5528/data/master.pid"
OCF_RESKEY_socket_default="/var/lib/mysql/mysql.sock"
...
...
參考：
http://clusterlabs.org/
http://clusterlabs.org/doc/en-US/Pacemaker/1.0/html/Pacemaker_Explained/index.html
http://freeloda.blog.51cto.com/2033581/1274533
http://freeloda.blog.51cto.com/2033581/1272417
----end---
