Web高可用負載均衡+動靜分離+session會話共享

總體架構：

lvs+keepalived+nginx+tomcat+redis

也是對之前實驗的總結

實驗環境：

主機名        IP                系統            應用
lvs1      192.168.14.211    centos7.6    lvs+keepalived（高可用）
lvs2      192.168.14.212    centos7.6    lvs+keepalived（高可用）
          192.168.14.216                    VIP（虛擬IP）
nginx1    192.168.14.213    centos7.6    nginx（處理靜態頁面）
nginx2    192.168.14.214    centos7.6    nginx（處理靜態頁面）
tomcat1   192.168.14.217    centos7.6    tomcat（處理動態頁面）
tomcat2   192.168.14.218    centos7.6    tomcat（處理動態頁面）
redis     192.168.14.219    centos7.6    redis（session會話共享）

一、lvs+keepalived部署

1、實驗環境關閉防火牆和selinux

#lvs1主機
[root@localhost ~]# hostnamectl set-hostname lvs1
[root@lvs1 ~]# systemctl stop firewalld
[root@lvs1 ~]# systemctl disable  firewalld
[root@lvs1 ~]# sed -i '/^SELINUX=/ s/enforcing/disabled/g' /etc/selinux/config 
[root@lvs1 ~]# setenforce 0

#lvs2主機
[root@localhost ~]# hostnamectl set-hostname lvs2
[root@lvs2 ~]# systemctl stop firewalld
[root@lvs2 ~]# systemctl disable firewalld
[root@lvs2 ~]# sed -i '/^SELINUX=/ s/enforcing/disabled/g' /etc/selinux/config
[root@lvs2 ~]# setenforce 0

2、安裝ipvs

[root@lvs1 ~]# yum -y install ipvsadm
[root@lvs2 ~]# yum -y install ipvsadm

3、安裝keepalived

[root@lvs1 ~]# yum install -y keepalived
[root@lvs2 ~]# yum install -y keepalived

4、修改keepalived配置文件

[root@lvs1 ~]# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
[root@lvs1 ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   router_id lvs1
}
vrrp_instance VI_1 {
    state MASTER
    interface ens32
    virtual_router_id 100
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.14.216
    }
}
virtual_server 192.168.14.216 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 5
    protocol TCP
    real_server 192.168.14.213 80 {
    weight 1
    TCP_CHECK {
       connect_timeout 10
       retry 3
       delay_before_retry 3
       connect_port 80 
       }
    }
     real_server 192.168.14.214 80 {
     weight 1
     TCP_CHECK {
       connect_timeout 10
       retry 3
       delay_before_retry 3
       connect_port 80
       }
     }
}

[root@lvs2 ~]# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
[root@lvs2 ~]# vi /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
   router_id lvs2
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens32
    virtual_router_id 100
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.14.216
    }
}
virtual_server 192.168.14.216 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 5
    protocol TCP
    real_server 192.168.14.213 80 {
    weight 1
    TCP_CHECK {
       connect_timeout 10
       retry 3
       delay_before_retry 3
       connect_port 80 
       }
    }
     real_server 192.168.14.214 80 {
     weight 1
     TCP_CHECK {
       connect_timeout 10
       retry 3
       delay_before_retry 3
       connect_port 80
       }
     }
}

5、啓動服務

[root@lvs1 ~]# systemctl start keepalived
[root@lvs1 ~]# systemctl enable keepalived

[root@lvs2 ~]# systemctl start keepalived
[root@lvs2 ~]# systemctl enable keepalived

6、查看vip，master節點纔有，backup節點沒有

二、nginx部署

1、實驗環境關閉防火牆和selinux

#nginx1主機
[root@localhost ~]# hostnamectl set-hostname nginx1
[root@nginx1 ~]# systemctl stop firewalld
[root@nginx1 ~]# systemctl disable firewalld
[root@nginx1 ~]# sed -i '/^SELINUX=/ s/enforcing/disabled/g' /etc/selinux/config
[root@nginx1 ~]# setenforce 0

#nginx2主機
[root@localhost ~]# hostnamectl set-hostname nginx2
[root@nginx2 ~]# systemctl stop firewalld
[root@nginx2 ~]# systemctl disable firewalld
[root@nginx2 ~]# sed -i '/^SELINUX=/ s/enforcing/disabled/g' /etc/selinux/config
[root@nginx2 ~]# setenforce 0

2、yum安裝

#添加yum源
cat  <<EOF>> /etc/yum.repos.d/nginx.repo 
[aliyun]
name=aliyun epel
baseurl=http://mirrors.aliyun.com/epel/7Server/x86_64/
gpgcheck=0
EOF
 
#net-tools是ifconfig使用需要
[root@nginx1 ~]# yum install -y nginx net-tools
[root@nginx2 ~]# yum install -y nginx net-tools

3、配置realserver.sh

打開Nginx所在服務器的“路由”功能、關閉“ARP查詢”功能並設置迴環ip，nginx01和nginx02配置如下

[root@nginx1 ~]# vi /etc/rc.d/init.d/realserver.sh
#!/bin/bash
    SNS_VIP=192.168.14.216
    /etc/rc.d/init.d/functions
    case "$1" in
    start)
        ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
        /sbin/route add -host $SNS_VIP dev lo:0
        echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
        echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
        echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
        echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
        sysctl -p >/dev/null 2>&1
        echo "RealServer Start OK"
        ;;
    stop)
        ifconfig lo:0 down
        route del $SNS_VIP >/dev/null 2>&1
        echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
        echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
        echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
        echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
        echo "RealServer Stoped"
        ;;
    *)
        echo "Usage: $0 {start|stop}"
        exit 1
    esac
    exit 0

此腳本用於節點服務器綁定 VIP ，並抑制響應 VIP 的 ARP 請求。這樣做的目的是爲了不讓關於 VIP 的 ARP 廣播時，節點服務器應答（ 因爲節點服務器都綁定了 VIP ，如果不做設置它們會應答，就會亂套 ）。

4、realserver.sh腳本授予執行權限

[root@nginx1 ~]# chmod u+x /etc/rc.d/init.d/realserver.sh
[root@nginx2 ~]# chmod u+x /etc/rc.d/init.d/realserver.sh

5、啓動服務

#不想看到第三行報錯刪除即可，不影響
[root@nginx1 ~]# /etc/rc.d/init.d/realserver.sh start 
/etc/rc.d/init.d/realserver.sh: line 3: /etc/rc.d/init.d/functions: Permission denied
RealServer Start OK

[root@nginx2 ~]# /etc/rc.d/init.d/realserver.sh start 
/etc/rc.d/init.d/realserver.sh: line 3: /etc/rc.d/init.d/functions: Permission denied
RealServer Start OK

6、查看vip

7、修改一下網頁顯示，並重啓服務

[root@nginx1 ~]# echo "nginx1 web" > /usr/share/doc/HTML/index.html
[root@nginx1 ~]# systemctl restart nginx
 
[root@nginx2 ~]# echo "nginx2 web" > /usr/share/doc/HTML/index.html
[root@nginx2 ~]# systemctl restart nginx 

8、壓力測試訪問vip

#使用AB工具壓力測試訪問
[root@localhost ~]# ab -c1000 -n1000 http://192.168.14.216/index.html
This is ApacheBench, Version 2.3 <$Revision: 1430300 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking 192.168.14.216 (be patient)
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Completed 700 requests
Completed 800 requests
Completed 900 requests
Completed 1000 requests
Finished 1000 requests


Server Software:        nginx/1.16.1
Server Hostname:        192.168.14.216
Server Port:            80

Document Path:          /index.html
Document Length:        11 bytes

Concurrency Level:      1000
Time taken for tests:   0.365 seconds
Complete requests:      1000
Failed requests:        0
Write errors:           0
Total transferred:      241000 bytes
HTML transferred:       11000 bytes
Requests per second:    2737.57 [#/sec] (mean)
Time per request:       365.287 [ms] (mean)
Time per request:       0.365 [ms] (mean, across all concurrent requests)
Transfer rate:          644.29 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:        1  134  11.5    134     154
Processing:    93  118  12.5    120     155
Waiting:        1   70  27.9     71     116
Total:        156  252  23.2    254     290

Percentage of the requests served within a certain time (ms)
  50%    254
  66%    266
  75%    273
  80%    275
  90%    283
  95%    286
  98%    288
  99%    289
 100%    290 (longest request)

9、查看服務器master節點（因爲是rr輪詢）

10、修改兩臺nginx配置文件，實現負載均衡和動靜分離。再重啓服務

    upstream tomcat {
        server 192.168.14.217:8080 max_fails=3 fail_timeout=20s weight=2;
        server 192.168.14.218:8080 max_fails=3 fail_timeout=20s weight=2;
    }

        location ~ \.(jsp|do)$ {
         proxy_pass http://tomcat;
        }

三、tomcat部署

1、實驗環境關閉防火牆和selinux

[root@localhost ~]# hostnamectl set-hostname tomcat1
[root@tomcat1 ~]# systemctl stop firewalld
[root@tomcat1 ~]# systemctl disable firewalld
[root@tomcat1 ~]# sed -i '/^SELINUX=/ s/enforcing/disabled/g' /etc/selinux/config
[root@tomcat1 ~]# setenforce 0


[root@localhost ~]# hostnamectl set-hostname tomcat2
[root@tomcat2 ~]# systemctl stop  firewalld
[root@tomcat2 ~]# systemctl disable firewalld
[root@tomcat2 ~]# sed -i '/^SELINUX=/ s/enforcing/disabled/g' /etc/selinux/config
[root@tomcat2 ~]# setenforce 0

2、配置java環境變量

[root@tomcat1 ~]# tar -zxvf jdk-8u141-linux-x64.tar.gz -C /usr/local/
[root@tomcat1 ~]# vi  /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_141/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar:$CLASSPATH
[root@tomcat1 ~]# source  /etc/profile


[root@tomcat2 ~]# tar -zxvf jdk-8u141-linux-x64.tar.gz -C /usr/local/
export JAVA_HOME=/usr/local/jdk1.8.0_141/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar:$CLASSPATH
[root@tomcat2 ~]# source /etc/profile

驗證

[root@tomcat1 ~]# java -version
java version "1.8.0_141"
Java(TM) SE Runtime Environment (build 1.8.0_141-b15)
Java HotSpot(TM) 64-Bit Server VM (build 25.141-b15, mixed mode)

3、下載tomcat（注意版本鏈接會更新）

[root@tomcat1 ~]# wget https://mirror.bit.edu.cn/apache/tomcat/tomcat-7/v7.0.104/bin/apache-tomcat-7.0.104.tar.gz

4、解壓並重命名

[root@tomcat1 ~]# tar -zxvf apache-tomcat-7.0.104.tar.gz -C /usr/local/
[root@tomcat1 ~]# mv /usr/local/apache-tomcat-7.0.104 /usr/local/tomcat7.0.104

[root@tomcat2 ~]# tar -zxvf apache-tomcat-7.0.104.tar.gz -C /usr/local/
[root@tomcat2 ~]# mv /usr/local/apache-tomcat-7.0.104 /usr/local/tomcat7.0.104

5、設置tomcat虛擬主機（兩臺都修改）

添加jvmRoute內容

修改默認虛擬主機，並將網站文件路徑指向/web/webapp1，在host段增加context段

[root@tomcat1 ~]# vim /usr/local/tomcat7.0.104/conf/server.xml

6、增加文檔目錄與測試文件

[root@tomcat1 ~]# mkdir -p /tomcat/webapp1
[root@tomcat1 ~]# vim /tomcat/webapp1/index.jsp
<%@page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<html>
<head>
<title>tomcat-1</title>
</head>
<body>
<h1><font color="red">Session serviced by tomcat</font></h1>
<table aligh="center" border="1">
<tr>
<td>Session ID</td>
<td><%=session.getId() %></td>
<% session.setAttribute("abc","abc");%>
</tr>
<tr>
<td>Created on</td>
<td><%= session.getCreationTime() %></td>
</tr>
</table>
</body>
<html>

[root@tomcat2 ~]# mkdir -p /tomcat/webapp2
[root@tomcat2 ~]# vim /tomcat/webapp2/index.jsp
<%@page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<html>
<head>
<title>tomcat-2</title>
</head>
<body>
<h1><font color="red">Session serviced by tomcat</font></h1>
<table aligh="center" border="1">
<tr>
<td>Session ID</td>
<td><%=session.getId() %></td>
<% session.setAttribute("abc","abc");%>
</tr>
<tr>
<td>Created on</td>
<td><%= session.getCreationTime() %></td>
</tr>
</table>
</body>
<html>

7、啓動tomcat

[root@tomcat1 ~]# /usr/local/tomcat7.0.104/bin/startup.sh
[root@tomcat2 ~]# /usr/local/tomcat7.0.104/bin/startup.sh

8、客戶端訪問VIP，驗證負載均衡

從上面的結果能看出兩次訪問，nginx把訪問請求分別分發給了後端的tomcat1和tomcat2，客戶端的訪問請求實現了負載均衡，但session  id不一樣(即：沒有實現session保持)，這樣的話，會給後端服務器造成很大的壓力。

四、redis部署

1、實驗環境關閉防火牆和selinux

[root@localhost ~]# hostnamectl set-hostname redis
[root@redis ~]# systemctl stop firewalld
[root@redis ~]# systemctl disable  firewalld
[root@redis ~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
[root@redis ~]# setenforce 0

2、下載redis源碼

[root@redis ~]# wget http://download.redis.io/releases/redis-3.2.3.tar.gz

3、解壓，編譯安裝

[root@redis ~]# tar  -zxvf redis-3.2.3.tar.gz
[root@redis ~]# yum install -y gcc gcc-c++ make
[root@redis ~]# cd redis-3.2.3
[root@redis redis-3.2.3]# make && make install

4、初始化redis，全部默認直接回車

[root@redis redis-3.2.3]# cd utils/
[root@redis utils]# ./install_server.sh

通過上面的安裝過程，我們可以看出redis初始化後redis配置文件爲

/etc/redis/6379.conf，日誌文件爲/var/log/redis_6379.log，數據文件dump.rdb存放到/var/lib/redis/6379目錄下，啓動腳本爲/etc/init.d/redis_6379。

5、現在我們要使用systemd，所以在 /etc/systems/system 下創建一個單位文件名字爲 redis_6379.service

[root@redis ~]# vi /etc/systemd/system/redis_6379.service
[Unit]
Description=Redison port 6379
[Service]
Type=forking
ExecStart=/etc/init.d/redis_6379 start
ExecStop=/etc/init.d/redis_6379 stop
[Install]
WantedBy=multi-user.target

6、啓動redis

[root@redis ~]# systemctl daemon-reload
[root@redis ~]# systemctl start  redis_6379.service
[root@redis ~]# systemctl enable   redis_6379.service

7、修改配置文件

[root@redis ~]# vim /etc/redis/6379.conf
[root@redis ~]# cat /etc/redis/6379.conf |grep -E 'bind|requirepass' |grep -v '#'
bind 127.0.0.1 192.168.14.219
requirepass pwd@123

8、重啓服務

[root@redis ~]# systemctl restart redis_6379.service

五、配置tomcat通過redis實現session同步

1、下載tomcat-redis-session-manager相應的jar包，主要有三個：

jedis-2.5.2.jar

commons-pool2-2.7.0.jar

tomcat7-redis-session-manager-2.0.0.jar

提供下載：https://download.csdn.net/download/tladagio/11727840

2、下載完成後拷貝到$TOMCAT_HOME/lib中

3、修改tomcat的context.xml

[root@tomcat1 ~]# vim /usr/local/tomcat7.0.104/conf/context.xml 

4、重啓tomcat

[root@tomcat1 ~]# /usr/local/tomcat7.0.104/bin/shutdown.sh 
Using CATALINA_BASE:   /usr/local/tomcat7.0.104
Using CATALINA_HOME:   /usr/local/tomcat7.0.104
Using CATALINA_TMPDIR: /usr/local/tomcat7.0.104/temp
Using JRE_HOME:        /usr/local/jdk1.8.0_141/
Using CLASSPATH:       /usr/local/tomcat7.0.104/bin/bootstrap.jar:/usr/local/tomcat7.0.104/bin/tomcat-juli.jar
[root@tomcat1 ~]# /usr/local/tomcat7.0.104/bin/startup.sh 
Using CATALINA_BASE:   /usr/local/tomcat7.0.104
Using CATALINA_HOME:   /usr/local/tomcat7.0.104
Using CATALINA_TMPDIR: /usr/local/tomcat7.0.104/temp
Using JRE_HOME:        /usr/local/jdk1.8.0_141/
Using CLASSPATH:       /usr/local/tomcat7.0.104/bin/bootstrap.jar:/usr/local/tomcat7.0.104/bin/tomcat-juli.jar
Tomcat started.
[root@tomcat1 ~]# 

5、tomcat2重複1-4步驟

六、查看動靜分離和session會話共享

1、訪問vip今天頁面

2、訪問vip動態頁面

3、查看redis緩存

至此，lvs+keepalived+nginx+tomcat+redis完成