corosync + pacemaker + drbd 實現mysql存儲的高可用（二）

 8. 定義資源

定義drbd資源

[root@node1 ~]# crm
crm(live)# ra
crm(live)ra# classes                        查看資源類型
heartbeat
lsb
ocf / heartbeat linbit pacemaker
stonith
crm(live)ra# list ocf linbit
drbd
crm(live)ra# cd
crm(live)# configure                          定義drbd資源
crm(live)configure# primitive webdrbd ocf:heartbeat:drbd params drbd_resource=web op start timeout=240 op stop timeout=100 op monitor role=Master interval=50s timeout=30s op monitor role=Slave interval=60s timeout=30s
crm(live)configure# show
node node1.yue.com \
    attributes standby="off"
node node2.yue.com
primitive webdrbd ocf:heartbeat:drbd \
    params drbd_resource="web" \
    op start interval="0" timeout="240" \
    op stop interval="0" timeout="100" \
    op monitor interval="50s" role="Master" timeout="30s" \
    op monitor interval="60s" role="Slave" timeout="30s"
property $id="cib-bootstrap-options" \
    dc-version="1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f" \
    cluster-infrastructure="openais" \
    expected-quorum-votes="2" \
    stonith-enabled="false" \
    no-quorum-policy="ignore"                    定義克隆資源
crm(live)configure# master MS_webdrbd webdrbd  meta master-max=1 master-node-max=1 clone-max=2 clone-node-max=1 notify=true
crm(live)configure# show
node node1.yue.com \
    attributes standby="off"
node node2.yue.com
primitive webdrbd ocf:heartbeat:drbd \
    params drbd_resource="web" \
    op start interval="0" timeout="240" \
    op stop interval="0" timeout="100" \
    op monitor interval="50s" role="Master" timeout="30s" \
    op monitor interval="60s" role="Slave" timeout="30s"
ms MS_webdrbd webdrbd \
    meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
property $id="cib-bootstrap-options" \
    dc-version="1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f" \
    cluster-infrastructure="openais" \
    expected-quorum-votes="2" \
    stonith-enabled="false" \
    no-quorum-policy="ignore"
crm(live)configure# commit                  提交
crm(live)configure# cd
crm(live)# status                           查看資源狀態
============
Last updated: Fri Aug 10 13:11:07 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
1 Resources configured.
============

Online: [ node1.yue.com node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node1.yue.com ]
     Slaves: [ node2.yue.com ]

定義文件系統資源：

crm(live)# configure
crm(live)configure# primitive WebFS ocf:heartbeat:Filesystem params device=/dev/drbd0 directory=/data fstype=ext3 op start timeout=60 op stop timeout=60
crm(live)configure# colocation WebFS_on_MS_webdrbd inf: WebFS MS_webdrbd:Master          排列約束
crm(live)configure# order WebFS_after_MS_webddrbd inf: MS_webdrbd:promote  WebFS:start   順序約束
crm(live)configure# show
node node1.yue.com \
    attributes standby="off"
node node2.yue.com
primitive WebFS ocf:heartbeat:Filesystem \
    params device="/dev/drbd0" directory="/data" fstype="ext3" \
    op start interval="0" timeout="60" \
    op stop interval="0" timeout="60"
primitive webdrbd ocf:heartbeat:drbd \
    params drbd_resource="web" \
    op start interval="0" timeout="240" \
    op stop interval="0" timeout="100" \
    op monitor interval="50s" role="Master" timeout="30s" \
    op monitor interval="60s" role="Slave" timeout="30s"
ms MS_webdrbd webdrbd \
    meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
colocation WebFS_on_MS_webdrbd inf: WebFS MS_webdrbd:Master
order WebFS_after_MS_webddrbd inf: MS_webdrbd:promote WebFS:start
property $id="cib-bootstrap-options" \
    dc-version="1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f" \
    cluster-infrastructure="openais" \
    expected-quorum-votes="2" \
    stonith-enabled="false" \
    no-quorum-policy="ignore"
crm(live)configure# commit                 提交

crm(live)configure# cd
crm(live)# status                        查看此時的資源狀態
============
Last updated: Fri Aug 10 13:13:54 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Online: [ node1.yue.com node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node1.yue.com ]
     Slaves: [ node2.yue.com ]
 WebFS  (ocf::heartbeat:Filesystem):    Started node1.yue.com
crm(live)# bye
bye

測試一下資源是否真的已經綁定在node1上：

[root@node1 ~]# drbd-overview                   查看資源情況
  0:web  Connected Primary/Secondary UpToDate/UpToDate C r---- /data ext3 479M 40M 415M 9%
[root@node1 ~]# ls /data/              查看一下數據目錄下是否存在數據文件
fstab    ib_logfile0  my.cnf  mysql-bin.000001  mysql-bin.index    node2.yue.com.err   test
ibdata1  ib_logfile1  mysql   mysql-bin.000002  node1.yue.com.err  performance_schema  testdb

從上面可以看出數據目錄下已經有數據文件了，我們再來測試一下mysql是否可以正常啓動
[root@node1 ~]# /etc/init.d/mysqld start
Starting MySQL...                                          [  OK  ]
[root@node1 ~]# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 1
Server version: 5.5.24-log MySQL Community Server (GPL)

Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> SHOW DATABASES;                   查看一下數據庫
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testdb             |
+--------------------+
5 rows in set (0.08 sec)

mysql> quit
Bye

下面我們來進行資源切換，測試一下資源切換是否正常

1）停止mysql

[root@node1 ~]# /etc/init.d/mysqld stop
Shutting down MySQL.                                       [  OK  ]

2）讓node1 處於standby

[root@node1 ~]# crm node standby                            讓執行命令的當前節點處於standby狀態
[root@node1 ~]# crm status
============
Last updated: Fri Aug 10 13:24:40 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Node node1.yue.com: standby
Online: [ node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node2.yue.com ]
     Stopped: [ webdrbd:0 ]
 WebFS  (ocf::heartbeat:Filesystem):    Started node2.yue.com

[root@node1 ~]# drbd-overview
  0:web  Unconfigured . . . .

3). 在node2上查看資源切換的結果

[root@node2 ~]# crm status
============
Last updated: Fri Aug 10 13:26:25 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Node node1.yue.com: standby
Online: [ node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node2.yue.com ]
     Stopped: [ webdrbd:0 ]
 WebFS  (ocf::heartbeat:Filesystem):    Started node2.yue.com     資源運行在node2 上
[root@node2 ~]# drbd-overview
  0:web  WFConnection Primary/Unknown UpToDate/DUnknown C r---- /data ext3 479M 40M 415M 9%

[root@node2 ~]# ls /data/
fstab    ib_logfile0  my.cnf  mysql-bin.000001  mysql-bin.000003  node1.yue.com.err  performance_schema  testdb
ibdata1  ib_logfile1  mysql   mysql-bin.000002  mysql-bin.index   node2.yue.com.err  test

讓node 1 重新上線，看一下會有什麼效果：

[root@node1 ~]# crm node online
[root@node1 ~]# drbd-overview
  0:web  Connected Secondary/Primary UpToDate/UpToDate C r----
[root@node1 ~]# crm status
============
Last updated: Fri Aug 10 13:30:13 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Online: [ node1.yue.com node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node2.yue.com ]
     Slaves: [ node1.yue.com ]
 WebFS  (ocf::heartbeat:Filesystem):    Started node2.yue.com      資源依然在node2 上並沒有自動切換回來

 此時我們再模擬node2 故障，資源應該會轉移到node1上去

[root@node2 ~]# crm node standby
[root@node2 ~]# crm status
============
Last updated: Fri Aug 10 13:35:46 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Node node2.yue.com: standby
Online: [ node1.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node1.yue.com ]
     Stopped: [ webdrbd:1 ]
 WebFS  (ocf::heartbeat:Filesystem):    Started node1.yue.com
[root@node2 ~]# drbd-overview
  0:web  Unconfigured . . . .


由上面的結果可以看到資源已經轉移到node1 上了
[root@node1 ~]# crm status
============
Last updated: Fri Aug 10 13:37:19 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Node node2.yue.com: standby
Online: [ node1.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node1.yue.com ]
     Stopped: [ webdrbd:1 ]
 WebFS  (ocf::heartbeat:Filesystem):    Started node1.yue.com
[root@node1 ~]# drbd-overview
  0:web  WFConnection Primary/Unknown UpToDate/DUnknown C r---- /data ext3 479M 40M 415M 9%
[root@node1 ~]# ls /data/
fstab    ib_logfile0  my.cnf  mysql-bin.000001  mysql-bin.000003  node1.yue.com.err  performance_schema  testdb
ibdata1  ib_logfile1  mysql   mysql-bin.000002  mysql-bin.index   node2.yue.com.err  test

4）若以上的測試都沒有問題，下面我們來配置mysql資源：

[root@node1 ~] # crm
crm(live)# configure
crm(live)configure# primitive web_mysql lsb:mysqld      定義mysql資源
crm(live)configure# primitive web_ip ocf:heartbeat:IPaddr2 params ip=172.16.45.2 nic=eth0 cidr_netmask=16 broadcast=172.16.45.255
                           此時額外定義一個vip，主要是爲mysql做爲後端負載均衡時的考慮，實驗中並未使用到相關的功能
crm(live)configure# show
node node1.yue.com \
        attributes standby="off"
node node2.yue.com \
        attributes standby="on"
primitive WebFS ocf:heartbeat:Filesystem \
        params device="/dev/drbd0" directory="/data" fstype="ext3" \
        op start interval="0" timeout="60" \
        op stop interval="0" timeout="60"
primitive web_ip ocf:heartbeat:IPaddr2 \
        params ip="172.16.45.2" nic="eth0" cidr_netmask="16" broadcast="172.16.45.255"
primitive web_mysql lsb:mysqld
primitive webdrbd ocf:heartbeat:drbd \
        params drbd_resource="web" \
        op start interval="0" timeout="240" \
        op stop interval="0" timeout="100" \
        op monitor interval="50s" role="Master" timeout="30s" \
        op monitor interval="60s" role="Slave" timeout="30s"
ms MS_webdrbd webdrbd \
        meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
colocation WebFS_on_MS_webdrbd inf: WebFS MS_webdrbd:Master
order WebFS_after_MS_webddrbd inf: MS_webdrbd:promote WebFS:start
property $id="cib-bootstrap-options" \
        dc-version="1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f" \
        cluster-infrastructure="openais" \
        expected-quorum-votes="2" \
        stonith-enabled="false" \
        no-quorum-policy="ignore"

crm(live)configure# commit       確認資源定義沒問題，然後提交讓其生效


還要定義排列線束和順序約束：
crm(live)configure# colocation mysql_on_ip inf: web_mysql web_ip
crm(live)configure# order mysql_after_FS inf: WebFS:start web_mysql     讓文件系統啓動成功後再啓動mysql
crm(live)configure# order ip_after_mysql inf: web_mysql web_ip
crm(live)configure# group cluster WebFS web_ip web_mysql     定義一個羣組，讓幾個資源在一起

crm(live)configure# show
node node1.yue.com \
    attributes standby="off"
node node2.yue.com \
    attributes standby="off"
primitive WebFS ocf:heartbeat:Filesystem \
    params device="/dev/drbd0" directory="/data" fstype="ext3" \
    op start interval="0" timeout="60" \
    op stop interval="0" timeout="60"
primitive web_ip ocf:heartbeat:IPaddr2 \
    params ip="172.16.45.2" nic="eth0" cidr_netmask="16" broadcast="172.16.45.255"
primitive web_mysql lsb:mysqld
primitive webdrbd ocf:heartbeat:drbd \
    params drbd_resource="web" \
    op start interval="0" timeout="240" \
    op stop interval="0" timeout="100" \
    op monitor interval="50s" role="Master" timeout="30s" \
    op monitor interval="60s" role="Slave" timeout="30s"
group cluster WebFS web_ip web_mysql
ms MS_webdrbd webdrbd \
    meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
colocation WebFS_on_MS_webdrbd inf: cluster MS_webdrbd:Master
order WebFS_after_MS_webddrbd inf: MS_webdrbd:promote cluster:start
property $id="cib-bootstrap-options" \
    dc-version="1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f" \
    cluster-infrastructure="openais" \
    expected-quorum-votes="2" \
    stonith-enabled="false" \
    no-quorum-policy="ignore"

crm(live)configure# commit        確認提交

來驗證下效果：

[root@node1 ~]# crm status
============
Last updated: Fri Aug 10 15:17:57 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Online: [ node1.yue.com node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node1.yue.com ]
     Slaves: [ node2.yue.com ]
 Resource Group: cluster
     WebFS  (ocf::heartbeat:Filesystem):    Started node1.yue.com
     web_ip (ocf::heartbeat:IPaddr2):   Started node1.yue.com
     web_mysql  (lsb:mysqld):   Started node1.yue.com

[root@node1 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:0c:29:e7:3d:5c brd ff:ff:ff:ff:ff:ff
    inet 172.16.45.11/16 brd 172.16.255.255 scope global eth0
    inet 172.16.45.2/16 brd 172.16.45.255 scope global secondary eth0   ip也已經綁定

[root@node1 ~]# drbd-overview
  0:web  WFConnection Primary/Unknown UpToDate/DUnknown C r---- /data ext3 479M 40M 415M 9%

看一下mysql的狀態：

[root@node1 ~]# /etc/init.d/mysqld status
MySQL running (2225)                                       [  OK  ]

[root@node1 ~]# mysql                連上mysql看一下
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 1
Server version: 5.5.24-log MySQL Community Server (GPL)

Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> SHOW DATABASES;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testdb             |
+--------------------+
5 rows in set (0.07 sec)

mysql> use testdb;
Database changed
mysql> CREATE TABLE student (Name char(20));            創建一張表
Query OK, 0 rows affected (0.14 sec)

mysql> DESC student;
+-------+----------+------+-----+---------+-------+
| Field | Type     | Null | Key | Default | Extra |
+-------+----------+------+-----+---------+-------+
| Name  | char(20) | YES  |     | NULL    |       |
+-------+----------+------+-----+---------+-------+
1 row in set (0.05 sec)

mysql> INSERT INTO student (Name) VALUE ('Redhat'),('Gentoo');   在表中插入一些數據
Query OK, 2 rows affected (0.01 sec)
Records: 2  Duplicates: 0  Warnings: 0

mysql> SELECT * FROM student;
+--------+
| Name   |
+--------+
| Redhat |
| Gentoo |
+--------+
2 rows in set (0.00 sec)

mysql> quit
Bye

四、模擬故障

讓node1 處於standby

[root@node1 ~]# crm node standby
[root@node1 ~]# crm status
============
Last updated: Fri Aug 10 15:20:12 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Node node1.yue.com: standby
Online: [ node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node2.yue.com ]
     Stopped: [ webdrbd:0 ]
 Resource Group: cluster
     WebFS  (ocf::heartbeat:Filesystem):    Started node2.yue.com
     web_ip (ocf::heartbeat:IPaddr2):   Started node2.yue.com
     web_mysql  (lsb:mysqld):   Stopped       mysql資源正在切換

[root@node1 ~]# crm status
============
Last updated: Fri Aug 10 15:20:22 2012
Stack: openais
Current DC: node1.yue.com - partition with quorum
Version: 1.1.5-1.1.el5-01e86afaaa6d4a8c4836f68df80ababd6ca3902f
2 Nodes configured, 2 expected votes
2 Resources configured.
============

Node node1.yue.com: standby
Online: [ node2.yue.com ]

 Master/Slave Set: MS_webdrbd [webdrbd]
     Masters: [ node2.yue.com ]
     Stopped: [ webdrbd:0 ]
 Resource Group: cluster
     WebFS  (ocf::heartbeat:Filesystem):    Started node2.yue.com
     web_ip (ocf::heartbeat:IPaddr2):   Started node2.yue.com
     web_mysql  (lsb:mysqld):   Started node2.yue.com      mysql已經切換完成

[root@node1 ~]# drbd-overview
  0:web  Unconfigured . . . .

[root@node2 ~]# drbd-overview
  0:web  StandAlone Primary/Unknown UpToDate/DUnknown r---- /data ext3 479M 40M 415M 9%

查看mysql

[root@node2 ~]# /etc/init.d/mysqld status
MySQL running (1274)                                       [  OK  ]
[root@node2 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
    link/ether 00:0c:29:d9:75:df brd ff:ff:ff:ff:ff:ff
    inet 172.16.45.12/16 brd 172.16.255.255 scope global eth0
    inet 172.16.45.2/16 brd 172.16.45.255 scope global secondary eth0

[root@node2 ~]# mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 1
Server version: 5.5.24-log MySQL Community Server (GPL)

Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.

Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.

Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

mysql> SHOW DATABASES;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| test               |
| testdb             |
+--------------------+
5 rows in set (0.09 sec)

mysql> use testdb
Database changed
mysql> SHOW TABLES;
+------------------+
| Tables_in_testdb |
+------------------+
| student          |
+------------------+
1 row in set (0.04 sec)

mysql> SELECT * FROM student;     我們剛纔在node1上創建的數據存在，
+--------+
| Name   |
+--------+
| Redhat |
| Gentoo |
+--------+
2 rows in set (0.02 sec)

mysql> quit
Bye

由此也說明了我們在一定的程度上保證了mysql數據的完整性，由於發生故障時資源切換和mysql啓動都需要一定的時間，所以還不能1保證數據是100%完整的！

另外在做實驗時曾出現瞭如下問題：

# /etc/init.d/mysqld status
MySQL is not running, but lock file (/var/lock/subsys/mysql[FAILED]

可能是我在做資源切換的時候忘記關閉mysql所導致的，此時可以將鎖文件刪除即可！

# cd /var/lock/subsys/
# rm -rf mysql