Centos7.2高可用集羣keepalived + LVS（DR模式）

拓撲圖

環境概述

      機器作用                IP地址
lvs1+keepalived Master   192.168.42.150
lvs2+keepalived Backup   192.168.42.149
web1                     192.168.42.148
web2                     192.168.42.147
VIP                      192.168.42.180

注意事項：
第一：確保client能夠直接訪問web服務器。因爲在LVS-DR模式下web服務器回覆client的時候，是直接回復給client的，不需要經過LVS，所以web服務器必須能夠和client的網絡互通。
第二，lvs服務器和web-server他們必須在同一個網段內，因爲LVS轉發包的時候，是直接修改了包目標的MAC地址，直接扔給了rs，基於MAC地址的修改是活動在OSI二層數據鏈路層的，工作在數據鏈路層的網絡設備就是交換機了，所以必須在一個交換機下面，也就是一個局域網內。
第三，爲啥抑制real-server的ARP，是這樣的，我們在DR模式要更改web-server的ARP的模式，arp_ignore爲1是說只回答目標IP地址是來訪網絡接口本地地址的ARP查詢請求，我們都在迴環接口上配置了一個VIP，當arp模式更改以後，那麼如果有誰在請求VIP的mac地址時，那麼那些web-server就會回答arp廣播報文了，只有lvs纔會，如果沒有更改arp模式，那麼大家都喊我是VIP,這就亂套了。

安裝部署

LVS配置

#打開路由轉發功能
[root@node1 ~]# echo "net.ipv4.ip_forward=1" > /etc/sysctl.conf
#使上述的配置文件生效
[root@node1 ~]# sysctl -p
net.ipv4.ip_forward = 1
#查看selinux是否關閉，未關閉則關閉
[root@node1 ~]# getenforce
Disabled
#關閉防火牆
[root@node1 ~]# systemctl stop firewalld
#安裝ipvsadm以及keepalived
yum install -y ipvsadm  keepalived
#配置keepalived.conf文件
[root@node1 keepalived]# vi /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {            # 全局設置
   notification_email {    # 設置報警郵件地址
        [email protected]
   }
   notification_email_from [email protected]   # 設置郵件的發送地址
   router_id MASTER       # 表示該臺服務的ID,備用節點修改爲BACKUP
}

vrrp_instance VI_1 {               # vrrp 實例
    state MASTER                   # 當前節點的角色，備用節點爲BACKUP
    interface ens33
    virtual_router_id 51           # master和backup的id一致
    priority 100                   #備用節點修改爲90
    advert_int 1                   # master和backup之間的檢測時間
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.42.180/32 dev ens33          # 設置虛擬IP
    }
}

virtual_server 192.168.42.180 80 {    # 設置虛擬服務器，ip端口以及花邊號用逗號隔開
    delay_loop 5         # 設置運行情況檢查時間，單位是秒
    lb_algo rr           # 負載算法，這裏是rr表示輪詢
    lb_kind DR           # LVS工作機制，這裏是DR模式
#    persistence_timeout 50    # 會話保持時間，單位是秒，這個選項對動態網頁非常有幫助，爲集羣系統中的session共享提供了一個很好的解決方案，有了這個會話>保持功能，用戶的請求會被一直分發到某個服務節點，知道超過這個會話的保持時間，需要注意的是，這個會話保持時間是最大無響應的超時時間，in other words，用戶>在動態頁面50秒以內沒有執行任何操作，那麼接下來的操作會重新調度到另一個節點上。如果一直在操作則不會影響
    protocol TCP   # 有TCP,UDP,好像還有哦http。

    real_server 192.168.42.148 80  {     #設定真實服務器地址以及端口，
        weight 1         # 設置權重，數字越低，調度的比例越小。
        HTTP_GET {      #以http模式檢查該服務器監控狀態
            url {    
              path /      # 檢測的網頁路徑
              digest 699d00db64614eb287931b977d5c047f      # 採用genhash命令獲取
            }
            connect_timeout 1   # 連接超時時間，
            connect_port 80      #連接的端口，
            nb_get_retry 3        # 表示重試次數，
            delay_before_retry 1   # 表示重試間隔。
        }
    }

    real_server 192.168.42.147 80  {
        weight 1
        HTTP_GET {
            url {
              path /
              digest 66ee606d5019d75f83836eeb295c6b6f
            }
            connect_timeout 2
            connect_port 80
            nb_get_retry 3
            delay_before_retry 1
        }
    }
}


#使用genhash獲取web服務的url的校驗碼
[root@node1 keepalived]# genhash -s 192.168.42.147 -p 80 -u /
MD5SUM = 66ee606d5019d75f83836eeb295c6b6f
[root@node1 keepalived]# genhash -s 192.168.42.148 -p 80 -u /
MD5SUM = 699d00db64614eb287931b977d5c047f

RealServer節點配置

#設置相關的抑制廣播通告的配置信息
[root@localhost ~]# cat /etc/sysctl.conf 
# System default settings live in /usr/lib/sysctl.d/00-system.conf.
# To override those settings, enter new settings here, or in an /etc/sysctl.d/<name>.conf file
#
# For more information, see sysctl.conf(5) and sysctl.d(5).
net.ipv4.conf.ens33.arp_ignore = 1
net.ipv4.conf.ens33.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2


#使得上面的配置生效
[root@localhost ~]# sysctl -p

#添加虛擬IP和路由
[root@localhost ~]# ifconfig lo:0 192.168.42.180 netmask 255.255.255.255 broadcast 192.168.42.180 up
[root@localhost ~]# route add -host 192.168.42.180 dev lo:0   #先不添加這路由，如果出現無法訪問的時候再添加也不遲

#安裝nginx，並啓動
yum install nginx -y
systemctl start nginx

啓動keepalived服務

#啓動keepalived
systemctl start keepalived

#查看相關的日誌信息，可以看到Master機器的日誌信息如下：
Nov  2 01:07:42 localhost systemd: Starting LVS and VRRP High Availability Monitor...
Nov  2 01:07:42 localhost Keepalived[2400]: Starting Keepalived v1.3.5 (03/19,2017), git commit v1.3.5-6-g6fa32f2
Nov  2 01:07:42 localhost Keepalived[2400]: Opening file '/etc/keepalived/keepalived.conf'.
Nov  2 01:07:42 localhost systemd: PID file /var/run/keepalived.pid not readable (yet?) after start.
Nov  2 01:07:42 localhost systemd: Started LVS and VRRP High Availability Monitor.
Nov  2 01:07:42 localhost Keepalived[2401]: Starting Healthcheck child process, pid=2402
Nov  2 01:07:42 localhost Keepalived[2401]: Starting VRRP child process, pid=2403
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: Registering Kernel netlink reflector
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: Registering Kernel netlink command channel
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: Registering gratuitous ARP shared channel
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: Opening file '/etc/keepalived/keepalived.conf'.
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: VRRP_Instance(VI_1) removing protocol VIPs.
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: Using LinkWatch kernel netlink reflector...
Nov  2 01:07:42 localhost Keepalived_vrrp[2403]: VRRP sockpool: [ifindex(2), proto(112), unicast(0), fd(10,11)]
Nov  2 01:07:42 localhost kernel: IPVS: Registered protocols (TCP, UDP, SCTP, AH, ESP)
Nov  2 01:07:42 localhost kernel: IPVS: Connection hash table configured (size=4096, memory=64Kbytes)
Nov  2 01:07:42 localhost kernel: IPVS: Creating netns size=2040 id=0
Nov  2 01:07:42 localhost kernel: IPVS: Creating netns size=2040 id=1
Nov  2 01:07:42 localhost kernel: IPVS: ipvs loaded.
Nov  2 01:07:42 localhost Keepalived_healthcheckers[2402]: Opening file '/etc/keepalived/keepalived.conf'.
Nov  2 01:07:42 localhost kernel: IPVS: [rr] scheduler registered.
Nov  2 01:07:42 localhost Keepalived_healthcheckers[2402]: Activating healthchecker for service [192.168.42.180]:80   # 後端real-server校驗成功，可以提供服務
Nov  2 01:07:42 localhost Keepalived_healthcheckers[2402]: Activating healthchecker for service [192.168.42.180]:80   # 後端real-server校驗成功，可以提供服務
Nov  2 01:07:43 localhost Keepalived_vrrp[2403]: VRRP_Instance(VI_1) Transition to MASTER STATE     # 設置本機爲master角色
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: VRRP_Instance(VI_1) Entering MASTER STATE
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: VRRP_Instance(VI_1) setting protocol VIPs.   #設置VIP到本機上
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.42.180
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:44 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:49 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:49 localhost Keepalived_vrrp[2403]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.42.180
Nov  2 01:07:49 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:49 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:49 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  2 01:07:49 localhost Keepalived_vrrp[2403]: Sending gratuitous ARP on ens33 for 192.168.42.180


#查看IP信息
[root@node1 keepalived]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:79:3e:93 brd ff:ff:ff:ff:ff:ff
    inet 192.168.42.150/24 brd 192.168.42.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.42.180/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe79:3e93/64 scope link 
       valid_lft forever preferred_lft forever

相關測試訪問

#在備用節點測試如下
[root@localhost ~]# curl http://192.168.42.180
web1
[root@localhost ~]# curl http://192.168.42.180
web2
[root@localhost ~]# curl http://192.168.42.180
web1
[root@localhost ~]# curl http://192.168.42.180
web2

#將主節點停掉，查看服務是否正常
[root@localhost ~]# systemctl stop keepalived

#查看備用節點的日誌信息
Nov  7 01:02:03 localhost Keepalived_vrrp[1275]: VRRP_Instance(VI_1) Transition to MASTER STATE
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: VRRP_Instance(VI_1) Entering MASTER STATE
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: VRRP_Instance(VI_1) setting protocol VIPs.
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.42.180
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:04 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:09 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:09 localhost Keepalived_vrrp[1275]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.42.180
Nov  7 01:02:09 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:09 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:09 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
Nov  7 01:02:09 localhost Keepalived_vrrp[1275]: Sending gratuitous ARP on ens33 for 192.168.42.180
由以上的信息可得，備用節點已經對外提供服務

#查看備用節點的網卡信息，看虛擬IP是否已經轉換
[root@localhost ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:a1:31:82 brd ff:ff:ff:ff:ff:ff
    inet 192.168.42.149/24 brd 192.168.42.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.42.180/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fea1:3182/64 scope link 
       valid_lft forever preferred_lft forever

#測試路由切換以後的訪問是否正常，在主節點測試
[root@localhost ~]# curl http://192.168.42.180
web2
[root@localhost ~]# curl http://192.168.42.180
web1
[root@localhost ~]# curl http://192.168.42.180
web2
[root@localhost ~]# curl http://192.168.42.180
web1


#將主節點的keepalived開啓之後，主節點又重新提供服務，備用節點重新回到備用狀態


#停掉某一個web1服務器，查看請求的內容
[root@localhost ~]# curl http://192.168.42.180
web2
[root@localhost ~]# curl http://192.168.42.180
web2
[root@localhost ~]# curl http://192.168.42.180
web2
可以看到，當停止web1服務器的時候，所有的請求都將調度至web2

Centos7.2高可用集羣keepalived + LVS（DR模式）

centos7.2簡易二進制方式安裝k8s v1.12.1 + flanneld

GIT的相關使用

centos7.2簡易二進制方式安裝k8s v1.12.1

Centos7.2高可用集羣keepalived + LVS（DR模式）

LVS的DR模式搭建

https://yachay.unat.edu.pe/blog/index.php?comment_area=format_blog&comment_component=blog&comment_co

linux以太網驅動總結