配置擴展存儲節點storage3
[root@storage3 ~]#yum install scsi-target-utils
[root@storage3 ~]#mkdir /iscsi
[root@storage3 ~]#dd if=/dev/zero of=/iscsi/disk-storage3 bs=1M count=2000
[root@storage3 ~]#vim /etc/tgt/targets.conf
<target iqn.2013-12.com.vir:storage3.target1>
backing-store /iscsi/disk-storage3
vendor_id storage3
product_id disk1
initiator-address 192.168.122.10
initiator-address 192.168.122.20
</target>
[root@storage3 ~]#/etc/init./tgtd start
集羣節點server1 & server2發現並登錄存儲storage3
[root@server1 ~]#iscsiadm -m discovery -t sendtargets -p 192.168.122.50:3260
[root@server1 ~]#iscsiadm -m node -T iqn.2013-12.com.vir:storage3.target1 -l
使用udev給存儲storage3創建鏈接
[root@server1 ~]#udevinfo -a -p /sys/block/sdd
[root@server1 ~]#vim /etc/udev/rules.d/80-iscsi.rules
SUBSYSTEM=="block",SYSFS{size}=="3919797",SYSFS{model}=="disk1",SYSFS{vendor}=="storage1",SYMLINK+="storage1-disk"
SUBSYSTEM=="block",SYSFS{size}=="3919797",SYSFS{model}=="disk1",SYSFS{vendor}=="storage2",SYMLINK+="storage2-disk"
SUBSYSTEM=="block",SYSFS{size}=="3919797",SYSFS{model}=="disk1",SYSFS{vendor}=="storage3",SYMLINK+="storage3-disk"
[root@server1 ~]#start_udev
在線擴容lviscsi到5G
[root@server1 ~]#pvcreate /dev/iscsi/storage3-disk
[root@server1 ~]#vgextend vgiscsi /dev/iscsi/storage3-disk
[root@server1 ~]#lvextend -L +3G /dev/vgiscsi/lviscsi
[root@server1 ~]#gfs2_grow -v /iscsi -->在線擴容
擴展應用服務器server3
分別修改存儲服務器storage1，storage2，storage3的配置文件targets.conf
[root@server1 ~]#vim /etc/tgt/targets.conf
......
initiator-address 192.168.122.10
initiator-address 192.168.122.20
initiator-address 192.168.122.60
......
[root@server1 ~]#tgt-admin --update ALL --force -->強制更新服務配置
在應用服務器server3上發現並登錄存儲
[root@server3 ~]#yum install iscsi-initiator-utils
[root@server3 ~]#/etc/init.d/iscsi start
[root@server3 ~]#iscsiadm -m discovery -t sendtargets -p 192.168.122.30:3260
[root@server3 ~]#iscsiadm -m discovery -t sendtargets -p 192.168.122.40:3260
[root@server3 ~]#iscsiadm -m discovery -t sendtargets -p 192.168.122.50:3260
[root@server3 ~]#iscsiadm -m node -T iqn.2013-12.com.vir:storage1.target1 -l
[root@server3 ~]#iscsiadm -m node -T iqn.2013-12.com.vir:storage2.target2 -l
[root@server3 ~]#iscsiadm -m node -T iqn.2013-12.com.vir:storage3.target3 -l
拷貝udev規則，並重啓udev
[root@server3 ~]#scp 192.168.122.10:/etc/udev/rules.d/80-iscsi.rules /etc/udev/rules.d/
[root@server3 ~]#start_udev
使server3加入集羣，並掛載存儲成功
[root@server3 ~]#pvscan
[root@server3 ~]#vgchange -ay vgiscsi
[root@server3 ~]#yum install cman
在server1上編輯cluster.conf，加入server3
[root@Sever1 ~]#vim /etc/cluster/cluster.conf
<?xml version="1.0" ?>
<cluster config_version="3" name="iscsi_cluster">
--> 修改配置文件，配置版本號+1(2->3）
<fence_daemon post_fail_delay="0" post_join_delay="3"/>
<clusternodes>
<clusternode name="server1.vir.com" nodeid="1" votes="1">
<fence/>
</clusternode>
<clusternode name="server2.vir.com" nodeid="2" votes="1">
<fence/>
</clusternode>
<clusternode name="server3.vir.com" nodeid="3" votes="1">
<fence/>
</clusternode>
</clusternodes>
<cman expected_votes="1" two_node="1"/>
<fencedevices/>
<rm>
<failoverdomains/>
<resources/>
</rm>
</cluster>
[root@server1 ~]#ccs_tool update /etc/cluster/cluster.conf
Config file updated from version 2 to 3
Update complete.
[root@server1 ~]#scp /etc/cluster/cluster.conf 192.168.122.60:/etc/cluster/
[root@server3 ~]#/etc/init.d/cman start
Starting cluster:
Loading modules... done
Mounting configfs... done
Starting ccsd... done
Starting cman... failed
cman not started: two_node set but there are more than 2 nodes /usr/sbin/cman_tool: aisexec daemon didn't start
[FAILED]
[root@server1 ~]#vim /etc/cluster/cluster.conf
<?xml version="1.0"?>
<cluster config_version="4" name="iscsi_cluster">
<fence_daemon post_fail_delay="0" post_join_delay="3"/>
<clusternodes>
<clusternode name="Server1.uplooking.com" nodeid="1" votes="1">
<fence/>
</clusternode>
<clusternode name="Server2.uplooking.com" nodeid="2" votes="1">
<fence/>
</clusternode>
<clusternode name="Server3.uplooking.com" nodeid="3" votes="1">
<fence/>
</clusternode>
</clusternodes>
<cman expected_votes="1" two_node="1"/> -->刪除two_node="1"
<fencedevices/>
<rm>
<failoverdomains/>
<resources/>
</rm>
</cluster>
[root@server1 ~]#ccs_tool update /etc/cluster/cluster.conf
Config file updated from version 3 to 4
Update complete.
[root@server3 ~]#/etc/init.d/cman start
Starting cluster:
Loading modules... done
Mounting configfs... done
Starting ccsd... done
Starting cman... done
Starting daemons... done
Starting fencing... done
[ OK ]
安裝並加載gfs2模塊，掛載gfs2文件系統
[root@server3 ~]#yum install gfs2-utils kmod-gfs
[root@server3 ~]#modprobe gfs2
[root@server3 ~]#mkdir /iscsi
[root@server3 ~]#mount -t gfs2 -o lockproto=lock_dlm /dev/vgiscsi/lviscsi /iscsi
/sbin/mount.gfs2: Too many nodes mounting filesystem, no free journals
由於之前創建GFS文件系統的時候，只指定了2個journal空間，現在添加server3後沒有空閒的journal空間分配，需要爲server3添加一個journal空間
[root@server1 ~]#gfs2_tool journals /iscsi
journal1 - 128MB
journal0 - 128MB
2 journal(s) found.
[root@server1 ~]#gfs2_jadd -j 1 /iscsi
Filesystem: /iscsi
Old Journals 2
New Journals 3
[root@Server3 ~]#mount -t gfs2 -o lockproto=lock_dlm /dev/vgiscsi/lviscsi /iscsi/