- 18.11 LVS DR模式搭建
- 18.12 keepalived + LVS
- 擴展
- haproxy+keepalived http://blog.csdn.net/xrt95050/article/details/40926255
- nginx、lvs、haproxy比較 http://www.csdn.net/article/2014-07-24/2820837
- keepalived中自定義腳本 vrrp_script http://my.oschina.net/hncscwc/blog/158746
- lvs dr模式只使用一個公網ip的實現方法 http://storysky.blog.51cto.com/628458/338726
# 18.11 LVS DR模式搭建
- 在生產環境用中的比較多的是DR模式，NAT模式有瓶頸，節省公網IP，對小公司來說公網IP也是花錢的，如果是配置的多臺機器，每臺機器都去配置一個公網IP就是很浪費資源的情況，而且當下公網IP越來越少；
- 另一種方案，搭建內部的lvs，全部都用內網，包括VIP也用內網，用一個公網IP做一個映射；公網的80端口映射到內網VIP的80端口，這樣可以節省IP
- DR模式搭建
- 準備工作
- 三臺機器
- dir aming-01（dir） 192.168.202.130 分發器，也叫調度器（簡寫爲dir）
- rs1 aming-02（rs1）192.168.202.132
- rs2 aming-03（rs2）192.168.202.133
- vip 192.168.202.200
- 因爲前一章節做了NAT模式，現在需要把兩臺rs 機器的網關 給改回來，原來的dir機器上配置的ens37的網卡就先不理他。
- 確保兩臺rs公網能ping通外網
- 先修改倆臺rs 機器的網關 原來倆臺機器的網關是192.168.202.2
- rs1
```
[root@aming-02 ~]# vi /etc/sysconfig/network-scripts/ifcfg-ens33
[root@aming-02 ~]# systemctl restart network.service
[root@aming-02 ~]#
```
- rs2
```
[root@aming-03 ~]# vi /etc/sysconfig/network-scripts/ifcfg-ens33
[root@aming-03 ~]# systemctl restart network.service
[root@aming-03 ~]#
```
- dir機器配置
- 在分發器服務器上，創建一個腳本
```
[root@aming-01 ~]# vim /usr/local/sbin/lvs_dr.sh
#! /bin/bash
echo 1 > /proc/sys/net/ipv4/ip_forward
ipv=/usr/sbin/ipvsadm
vip=192.168.202.200
rs1=192.168.202.132
rs2=192.168.202.133
#注意這裏的網卡名字
ifdown ens33
ifup ens33
ifconfig ens33:2 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev ens33:2
$ipv -C
$ipv -A -t $vip:80 -s rr
$ipv -a -t $vip:80 -r $rs1:80 -g -w 1
$ipv -a -t $vip:80 -r $rs2:80 -g -w 1
~
:wq
[root@aming-01 ~]# vim /usr/local/sbin/lvs_dr.sh
[root@aming-01 ~]#
```
- 啓動下腳本
```
[root@aming-01 ~]# sh /usr/local/sbin/lvs_dr.sh
成功斷開設備 'ens33'。
成功激活的連接（D-Bus 激活路徑：/org/freedesktop/NetworkManager/ActiveConnection/9）
[root@aming-01 ~]#
```
- 倆個rs機器上也要配置腳本
- 兩臺rs上也編寫腳本 vim /usr/local/sbin/lvs_rs.sh//內容如下
- 先在rs1 aming-02上
```
[root@aming-02 ~]# vi /usr/local/sbin/lvs_rs.sh
#/bin/bash
vip=192.168.202.200
#把vip綁定在lo上，是爲了實現rs直接把結果返回給客戶端
ifdown lo
ifup lo
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
#以下操作爲更改arp內核參數，目的是爲了讓rs順利發送mac地址給客戶端
#參考文檔www.cnblogs.com/lgfeng/archive/2012/10/16/2726308.html
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
:wq
[root@aming-02 ~]# vi /usr/local/sbin/lvs_rs.sh
[root@aming-02 ~]# sh /usr/local/sbin/lvs_rs.sh
[root@aming-02 ~]#
[root@aming-02 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet 192.168.202.200/32 brd 192.168.202.200 scope global lo:0
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:58:33:e6 brd ff:ff:ff:ff:ff:ff
inet 192.168.202.132/24 brd 192.168.202.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.202.152/24 brd 192.168.202.255 scope global secondary ens33:0
valid_lft forever preferred_lft forever
inet6 fe80::ecdd:28b7:612b:cb7/64 scope link
valid_lft forever preferred_lft forever
[root@aming-02 ~]#
```
- rs2 aming-03 也是一樣創建一個腳本添加如下內容
```
[root@aming-03 ~]# vi /usr/local/sbin/lvs_rs.sh
#/bin/bash
vip=192.168.202.200
#把vip綁定在lo上，是爲了實現rs直接把結果返回給客戶端
ifdown lo
ifup lo
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
#以下操作爲更改arp內核參數，目的是爲了讓rs順利發送mac地址給客戶端
#參考文檔www.cnblogs.com/lgfeng/archive/2012/10/16/2726308.html
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
~
:wq
執行腳本
[root@aming-03 ~]# vi /usr/local/sbin/lvs_rs.sh
[root@aming-03 ~]# sh /usr/local/sbin/lvs_rs.sh
[root@aming-03 ~]#
```
- 可以route -n 查看下
```
[root@aming-03 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.202.2 0.0.0.0 UG 100 0 0 ens33
192.168.202.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
192.168.202.200 0.0.0.0 255.255.255.255 UH 0 0 0 lo
[root@aming-03 ~]#
[root@aming-03 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet 192.168.202.200/32 brd 192.168.202.200 scope global lo:0
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:9c:2b:f0 brd ff:ff:ff:ff:ff:ff
inet 192.168.202.133/24 brd 192.168.202.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.202.153/24 brd 192.168.202.255 scope global secondary ens33:0
valid_lft forever preferred_lft forever
inet6 fe80::4500:6d42:8612:4e53/64 scope link
valid_lft forever preferred_lft forever
inet6 fe80::ecdd:28b7:612b:cb7/64 scope link tentative dadfailed
valid_lft forever preferred_lft forever
[root@aming-03 ~]#
```
- 去分發器dir 上看下
```
[root@aming-01 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:2e:28:f2 brd ff:ff:ff:ff:ff:ff
inet 192.168.202.130/24 brd 192.168.202.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.202.200/32 brd 192.168.202.200 scope global ens33:2
valid_lft forever preferred_lft forever
inet 192.168.202.150/24 brd 192.168.202.255 scope global secondary ens33:0
valid_lft forever preferred_lft forever
inet6 fe80::ddac:89a0:52f8:d08d/64 scope link
valid_lft forever preferred_lft forever
inet6 fe80::4500:6d42:8612:4e53/64 scope link tentative dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::ecdd:28b7:612b:cb7/64 scope link tentative dadfailed
valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:2e:28:fc brd ff:ff:ff:ff:ff:ff
inet 192.168.142.147/24 brd 192.168.142.255 scope global ens37
valid_lft forever preferred_lft forever
inet 192.168.142.128/24 brd 192.168.142.255 scope global secondary dynamic ens37
valid_lft 1235sec preferred_lft 1235sec
inet6 fe80::20c:29ff:fe2e:28fc/64 scope link
valid_lft forever preferred_lft forever
[root@aming-01 ~]#
```
- 分別在機器上執行這些腳本
- 測試
- 去windows 瀏覽器訪問下192.168.202.200
- 
- 刷新沒有效果，主要是瀏覽器上有緩存，那就在dir機器上crul試試
```
[root@aming-01 ~]# iptables -nvL
Chain INPUT (policy ACCEPT 4266 packets, 452K bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 479 packets, 47671 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 17593 packets, 968K bytes)
pkts bytes target prot opt in out source destination
[root@aming-01 ~]# iptables -t nat -nvL
Chain PREROUTING (policy ACCEPT 217 packets, 32912 bytes)
pkts bytes target prot opt in out source destination
Chain INPUT (policy ACCEPT 49 packets, 9980 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 100 packets, 9178 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 9 packets, 2456 bytes)
pkts bytes target prot opt in out source destination
115 8744 MASQUERADE all -- * * 192.168.202.0/24 0.0.0.0/0
[root@aming-01 ~]# iptables -t nat -F
[root@aming-01 ~]# curl http://192.168.202.200/
^C
^C
[root@aming-01 ~]#
```
- 不能訪問的原因是倆邊的機器都有vip，在本機上訪問不行，最好是再開一個虛擬機，或者自己的windows瀏覽器訪問下
- 
```
[root@aming-01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.202.200:80 rr
-> 192.168.202.132:80 Route 1 1 3
-> 192.168.202.133:80 Route 1 1 1
[root@aming-01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.202.200:80 rr
-> 192.168.202.132:80 Route 1 1 2
-> 192.168.202.133:80 Route 1 1 1
[root@aming-01 ~]#
```
# 18.12 keepalived lvs
- 完整架構需要兩臺服務器（角色爲dir）分別安裝keepalived軟件，目的是實現高可用，但keepalived本身也有負載均衡的功能，所以本次實驗可以只安裝一臺keepalived
- 爲什麼需要把keepalived 加到lvs 中的目的是什麼
- 第一個原因：lvs有個分發器角色，如果宕掉以後，後端的rs就沒有辦法繼續使用，所以需要用keepalived做一個高可用，
- 第二個原因：在使用lvs的時候，當後端有一臺rs機器宕機時，lvs照樣會分發數據到這臺宕機機器，這是就會出現訪問無效的情況，說明lvs並不聰明；這時使用keepalived，就可以保證集羣中其中一臺rs宕機了，web還能正常提供，不會出現用戶訪問時無效鏈接的結果；一般這種架構，肯定是2臺keepalived；
- 因爲keepalived內置了ipvsadm的功能，所以不再需要安裝ipvsadm的包，也不用再編寫和執行.sh腳本
- 準備工作
- 三臺機器分別爲：
- dir（安裝keepalived）202.130
- rs1 202.132
- rs2 202.133
- vip 202.200
- 卸載掉ipvmsamd 清空ipvsadm規則
ipvsadm -C
- 查看一下ipvsadm的規則
```
[root@aming-01 ~]# ipvsadm -C
[root@aming-01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@aming-01 ~]#
```
- 清除之前配置的IP
sytemctl restart network
- 編輯keepalived配置文件 vim /etc/keepalived/keepalived.conf
- 內容請到https://coding.net/u/aminglinux/p/aminglinux-book/git/blob/master/D21Z/lvs_keepalived.conf 獲取
- 需要更改裏面的ip信息
- 先嚐試關閉rs2 aming-03的 nginx
```
[root@aming-03 ~]# systemctl stop nginx
[root@aming-03 ~]# ps aux |grep nginx
root 4956 0.0 0.0 112680 980 pts/0 S+ 23:59 0:00 grep --color=auto nginx
[root@aming-03 ~]#
```
-
```
[root@aming-01 ~]# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
#備用服務器上爲 BACKUP
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
192.168.202.200
virtual_router_id 51
#備用服務器上爲90
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass aminglinux
}
virtual_ipaddress {
192.168.202.200
}
}
virtual_server 192.168.202.200 80 {
#(每隔10秒查詢realserver狀態)
delay_loop 10
#(lvs 算法)
lb_algo wlc
#(DR模式)
lb_kind DR
#(同一IP的連接60秒內被分配到同一臺realserver)
persistence_timeout 0
#(用TCP協議檢查realserver狀態)
protocol TCP
real_server 192.168.202.132 80 {
#(權重)
weight 100
TCP_CHECK {
#(10秒無響應超時)
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.202.133 80 {
weight 100
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
:wq
[root@aming-01 ~]# vim /etc/keepalived/keepalived.conf
```
- 開啓keepalived 服務,看看進程
```
[root@aming-01 ~]# systemctl start keepalived
[root@aming-01 ~]#
[root@aming-01 ~]# ps aux |grep keep
root 25759 0.0 0.0 112680 980 pts/2 R+ 23:47 0:00 grep --color=auto keep
root 124427 0.0 0.1 120720 1400 ? Ss 20:01 0:01 /usr/sbin/keepalived -D
root 124428 0.0 0.2 120720 2756 ? S 20:01 0:01 /usr/sbin/keepalived -D
root 124429 0.0 0.2 124976 2760 ? S 20:01 0:10 /usr/sbin/keepalived -D
[root@aming-01 ~]#
```
- 看下ip
```
[root@aming-01 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:2e:28:f2 brd ff:ff:ff:ff:ff:ff
inet 192.168.202.130/24 brd 192.168.202.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.202.200/32 brd 192.168.202.200 scope global ens33:2
valid_lft forever preferred_lft forever
inet 192.168.202.150/24 brd 192.168.202.255 scope global secondary ens33:0
valid_lft forever preferred_lft forever
inet6 fe80::ddac:89a0:52f8:d08d/64 scope link
valid_lft forever preferred_lft forever
inet6 fe80::4500:6d42:8612:4e53/64 scope link tentative dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::ecdd:28b7:612b:cb7/64 scope link tentative dadfailed
valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:2e:28:fc brd ff:ff:ff:ff:ff:ff
inet 192.168.142.147/24 brd 192.168.142.255 scope global ens37
valid_lft forever preferred_lft forever
inet 192.168.142.128/24 brd 192.168.142.255 scope global secondary dynamic ens37
valid_lft 1433sec preferred_lft 1433sec
inet6 fe80::20c:29ff:fe2e:28fc/64 scope link
valid_lft forever preferred_lft forever
[root@aming-01 ~]#
```
- 查看下啓動後的規則
```
[root@aming-01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.202.200:80 wlc
-> 192.168.202.132:80 Route 100 0 0
-> 192.168.202.133:80 Route 100 0 0
[root@aming-01 ~]#
```
- 停掉keepalived服務 再看下就沒有規則了
```
[root@aming-01 ~]# systemctl stop keepalived
[root@aming-01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@aming-01 ~]#
```
- 再啓動keepalived 再看就有了
```
[root@aming-01 ~]# systemctl start keepalived
[root@aming-01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.202.200:80 wlc
-> 192.168.202.132:80 Route 100 0 0
-> 192.168.202.133:80 Route 100 0 0
[root@aming-01 ~]#
```
- 同時還需要做兩點
- 1.打開dir機器的端口轉發
```
echo 1 > /proc/sys/net/ipv4/ip_forward //打開端口轉發
- 2.運行前一章在rs機器上創建的lvs_rs.sh腳本
- #把vip綁定在lo上，是爲了實現rs直接把結果返回給客戶端
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
- #以下操作爲更改arp內核參數，目的是爲了讓rs順利發送mac地址給客戶端
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
```
- 總結：
- keepalived 有一個比較好的功能，可以在一臺rs宕機的時候，及時把他踢出 ipvsadm 集羣，將不再發送數據包給，也就很好的避免的訪問無連接的情況發送18.11 LVS DR模式搭建 18.12 keepalived + LVS
發表評論
所有評論
還沒有人評論，想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.