提示：關於nginx 和tomcat 己經在前面的博文己在介紹過了
第一部分：nginx反向代理tomcat
一、軟件及環境
軟件
| 系統 | 角色 | 用途 | 安裝的軟件 | ip地址 |
| Centos6.5x86_64 | nginx | 反向代理用戶請求 | nginx | 172.16.249.210 |
| Centos6.5x86_64 | tomcat1 | 處理jsp請求 | jdk+tomcat | 172.16.249.101 |
| Centos6.5x86_64 | tomcat2 | 172.16.249.100 | ||
| Centos6.5x86_64 | mog(ilefs/store) | 分佈式存儲靜態數據 | mogilefs | 172.16.249.129 |
| Centos6.5x86_64 | mog(ilefs/store)+mariadb | 172.16.249.128 |
拓撲圖
二、實現過程
1、nginx安裝
配置文件如下：
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream mogfs_cluster {
server 172.16.249.128:7001;
server 172.16.249.129:7001;
}
upstream jsp_server {
server 172.16.249.100:8080;
server 172.16.249.101:8080;
}
server {
listen 80;
# server_name localhost;
# location / {
# root html;
# index index.html index.htm;
# }
location ~* ^(/images/.*)$ {
mogilefs_tracker mogfs_cluster;
mogilefs_domain images;
mogilefs_noverify on;
mogilefs_pass $1 {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location ~* ^(/text/.*)$ {
mogilefs_tracker mogfs_cluster;
mogilefs_domain text;
mogilefs_noverify on;
mogilefs_pass $1 {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location ~* (\.jsp|do)$ {
proxy_pass http://jsp_server;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}tomcat1配置文件如下（server.xml）
<?xml version='1.0' encoding='utf-8'?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Note: A "Server" is not itself a "Container", so you may not
define subcomponents such as "Valves" at this level.
Documentation at /docs/config/server.html
-->
<Server port="8005" shutdown="SHUTDOWN">
<!-- Security listener. Documentation at /docs/config/listeners.html
<Listener className="org.apache.catalina.security.SecurityListener" />
-->
<!--APR library loader. Documentation at /docs/apr.html -->
<Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
<!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
<Listener className="org.apache.catalina.core.JasperListener" />
<!-- Prevent memory leaks due to use of particular java/javax APIs-->
<Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
<Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
<Listener className="org.apache.catalina.core.ThreadLocalLeakPreventionListener" />
<!-- Global JNDI resources
Documentation at /docs/jndi-resources-howto.html
-->
<GlobalNamingResources>
<!-- Editable user database that can also be used by
UserDatabaseRealm to authenticate users
-->
<Resource name="UserDatabase" auth="Container"
type="org.apache.catalina.UserDatabase"
description="User database that can be updated and saved"
factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
pathname="conf/tomcat-users.xml" />
</GlobalNamingResources>
<!-- A "Service" is a collection of one or more "Connectors" that share
a single "Container" Note: A "Service" is not itself a "Container",
so you may not define subcomponents such as "Valves" at this level.
Documentation at /docs/config/service.html
-->
<Service name="Catalina">
<!--The connectors can use a shared executor, you can define one or more named thread pools-->
<!--
<Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
maxThreads="150" minSpareThreads="4"/>
-->
<!-- A "Connector" represents an endpoint by which requests are received
and responses are returned. Documentation at :
Java HTTP Connector: /docs/config/http.html (blocking & non-blocking)
Java AJP Connector: /docs/config/ajp.html
APR (HTTP/AJP) Connector: /docs/apr.html
Define a non-SSL HTTP/1.1 Connector on port 8080
-->
<Connector port="8080" protocol="HTTP/1.1"
connectionTimeout="20000"
redirectPort="8443" />
<!-- A "Connector" using the shared thread pool-->
<!--
<Connector executor="tomcatThreadPool"
port="8080" protocol="HTTP/1.1"
connectionTimeout="20000"
redirectPort="8443" />
-->
<!-- Define a SSL HTTP/1.1 Connector on port 8443
This connector uses the JSSE configuration, when using APR, the
connector should be using the OpenSSL style configuration
described in the APR documentation -->
<!--
<Connector port="8443" protocol="HTTP/1.1" SSLEnabled="true"
maxThreads="150" scheme="https" secure="true"
clientAuth="false" sslProtocol="TLS" />
-->
<!-- Define an AJP 1.3 Connector on port 8009 -->
<Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
<!-- An Engine represents the entry point (within Catalina) that processes
every request. The Engine implementation for Tomcat stand alone
analyzes the HTTP headers included with the request, and passes them
on to the appropriate Host (virtual host).
Documentation at /docs/config/engine.html -->
<!-- You should set jvmRoute to support load-balancing via AJP ie :
<Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
-->
<Engine name="Catalina" defaultHost="localhost">
<!--For clustering, please take a look at documentation at:
/docs/cluster-howto.html (simple how to)
/docs/config/cluster.html (reference documentation) -->
<!--
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
-->
<!-- Use the LockOutRealm to prevent attempts to guess user passwords
via a brute-force attack -->
<Realm className="org.apache.catalina.realm.LockOutRealm">
<!-- This Realm uses the UserDatabase configured in the global JNDI
resources under the key "UserDatabase". Any edits
that are performed against this UserDatabase are immediately
available for use by the Realm. -->
<Realm className="org.apache.catalina.realm.UserDatabaseRealm"
resourceName="UserDatabase"/>
</Realm>
<Host name="www.essun.org" appBase="webapps"
unpackWARs="true" autoDeploy="true">
<Context path="" docBase="essun" />
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="essun_access_log." suffix=".txt"
pattern="%h %l %u %t "%r" %s %b" />
</Host>
<Host name="localhost" appBase="webapps"
unpackWARs="true" autoDeploy="true">
<!-- SingleSignOn valve, share authentication between web applications
Documentation at: /docs/config/valve.html -->
<!--
<Valve className="org.apache.catalina.authenticator.SingleSignOn" />
-->
<!-- Access log processes all example.
Documentation at: /docs/config/valve.html
Note: The pattern used is equivalent to using pattern="common" -->
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="localhost_access_log." suffix=".txt"
pattern="%h %l %u %t "%r" %s %b" />
</Host>
</Engine>
</Service>
</Server>站點目錄文件如下
[root@essun tomcat]# cd webapps/essun/
[root@essun essun]# ls
index.jsp WEB-INF
[root@essun essun]# ll WEB-INF/
total 8
drwxr-xr-x 2 root root 4096 May 2 11:07 classes
drwxr-xr-x 2 root root 4096 May 2 11:07 lib
[root@essun essun]# cat index.jsp
<%@ page language="java" %>
<html>
<head><title>essun.node2</title></head>
<body> <h1><font color="red">essun.node2.org</font></h1>
<table align="centre" border="1">
<tr>
<td>Session ID</td>
<% session.setAttribute("essun.org","essun.org"); %>
<td><%= session.getId() %></td>
</tr>
<tr>
<td>Created on</td>
<td><%= session.getCreationTime() %></td>
</tr>
</table>
<p>
<a href="http://172.16.249.210/images/4.jpg">"Picture"</a></p>
<% out.println("<img src='http://172.16.249.210/images/3.jpg'>"); %>
</body>
</html>tomcat2 與此處的配置相同，僅 index.jsp有所不同
[root@essun essun]# cat index.jsp
<%@ page language="java" %>
<html>
<head><title>essun.node1</title></head>
<body> <h1><font color="blue">essun.node1.org</font></h1>
<table align="centre" border="1">
<tr>
<td>Session ID</td>
<% session.setAttribute("essun.org","essun.org"); %>
<td><%= session.getId() %></td>
</tr>
<tr>
<td>Created on</td>
<td><%= session.getCreationTime() %></td>
</tr>
</table>
<p>
<a href="http://172.16.249.210/images/4.jpg">"picture"</a></p>
<% out.println("<img src='http://172.16.249.210/images/3.jpg'>"); %>
</body>
</html>配置文件中只要修改對應的ip地址即可
4、測試
訪問nginx
刷新後的結果
己經通過nginx中的proxy_pass爲tomcat反向代理
注：
<a href="http://172.16.249.210/images/4.jpg">"picture"</a></p><% out.println("<img src='http://172.16.249.210/images/3.jpg'>"); %>這兩處的url都是nginx反向代理mogilefs的url.
第二部分、實現session會話同步（DeltaManager方式）
tomcat會話管理
會話分類
標準會話管理器和持久會話管理器
標準會話管理器(StandardManager)
<Manager className="org.apache.catalina.session.StandardManager"
maxInactiveInterval="7200"/>默認保存於$CATALINA_HOME/work/Catalina/<hostname>/<webapp-name>/下的SESSIONS.ser文件中。
maxActiveSessions：最多允許的活動會話數量，默認爲-1，表示不限制；
maxInactiveInterval：非活動的會話超時時長，默認爲60s；
pathname：會話文件的保存目錄； 持久會話管理器(PersistentManager)：將會話數據保存至持久存儲中，並且能在服務器意外中止後重新啓動時重新加載這些會話信息。持久會話管理器支持將會話保存至文件存儲(FileStore)或JDBC存儲(JDBCStore)中。
保存至文件中的示例
<Manager className="org.apache.catalina.session.PersistentManager"
saveOnRestart="true">
<Store className="org.apache.catalina.session.FileStore"
directory="/data/tomcat-sessions"/>
</Manager>每個用戶的會話會被保存至directory指定的目錄中的文件中，文件名爲.session，並通過後臺線程每隔一段時間(checkInterval參數定義，默認爲60秒)檢查一次超時會話。
保存至JDBCStore中的示例
<Manager className="org.apache.catalina.session.PersistentManager"
saveOnRestart="true">
<Store className="org.apache.catalina.session.JDBCStore"
driverName="com.mysql.jdbc.Driver"
connectionURL="jdbc:mysql://localhost:3306/mydb?user=jb;password=pw"/>
</Manager>Manger對象用於實現HTTP會話管理的功能，Tomcat6中有4種Manger的實現：
StandardManagerTomcat7的默認會話管理器，用於非集羣環境中對單個處於運行狀態的Tomcat實例會話進行管理。當Tomcat關閉時，這些會話相關的數據會被寫入磁盤上的一個名叫SESSION.ser的文件，並在Tomcat下次啓動時讀取此文件。
PersistentManager當一個會話長時間處於空閒狀態時會被寫入到swap會話對象，這對於內存資源比較吃緊的應用環境來說比較有用。
DeltaManager用於Tomcat集羣的會話管理器，它通過將改變了的會話數據同步給集羣中的其它節點實現會話複製。這種實現會將所有會話的改變同步給集羣中的每一個節點，也是在集羣環境中用得最多的一種實現方式。
BackupManager用於Tomcat集羣的會話管理器，與DeltaManager不同的是，某節點會話的改變只會同步給集羣中的另一個而非所有節點。
注 ：
負載均衡，且實現會話綁定要注意給每個tomcat實例的egine容器一個jvmRoute屬性！此名稱要跟前端調度模塊使用名稱保持一致！另外，在mod_proxy實現負載均衡的會話綁定時，還要使用sticksession=JSESSIONID（字符要大寫）！
1、分別打開tomcat1和tomcat2下conf中server.xml
<Server port="8005" shutdown="SHUTDOWN"> <!-- Define a non-SSL HTTP/1.1 Connector on port 8080 --> <Connector port="8080" maxHttpHeaderSize="8192" maxThreads="150" minSpareThreads="25" maxSpareThreads="75" enableLookups="false" redirectPort="8443" acceptCount="100" connectionTimeout="20000" disableUploadTimeout="true" /> <!-- Define an AJP 1.3 Connector on port 8009 --> <Connector port="8009" enableLookups="false" redirectPort="8443" protocol="AJP/1.3" />
找到<Engine name="Catalina" defaultHost="localhost">
Tomcat1下的修改爲
<Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat1">
Tomcat2下的修改爲<Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat2">
2、修改tomcat1中的Cluster className tomcat1找到 修改爲
<!--
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
-->
#在這以下爲添加內容
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"
channelSendOptions="8">
<Manager className="org.apache.catalina.ha.session.DeltaManager"
expireSessionsOnShutdown="false"
notifyListenersOnReplication="true"/>
<Channel className="org.apache.catalina.tribes.group.GroupChannel">
<Membership className="org.apache.catalina.tribes.membership.McastService"
address="228.0.0.4"
port="45564"
frequency="500"
dropTime="3000"/>
<Receiver className="org.apache.catalina.tribes.transport.nio.NioReceiver"
address="172.16.249.101"
port="4000"
autoBind="100"
selectorTimeout="5000"
maxThreads="6"/>
<Sender className="org.apache.catalina.tribes.transport.ReplicationTransmitter">
<Transport className="org.apache.catalina.tribes.transport.nio.PooledParallelSender"/>
</Sender>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.TcpFailureDetector"/>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.MessageDispatch15Interceptor"/>
</Channel>
<Valve className="org.apache.catalina.ha.tcp.ReplicationValve"
filter=""/>
<Valve className="org.apache.catalina.ha.session.JvmRouteBinderValve"/>
<Deployer className="org.apache.catalina.ha.deploy.FarmWarDeployer"
tempDir="/tmp/war-temp/"
deployDir="/tmp/war-deploy/"
watchDir="/tmp/war-listen/"
watchEnabled="false"/>
<ClusterListener className="org.apache.catalina.ha.session.JvmRouteSessionIDBinderListener"/>
<ClusterListener className="org.apache.catalina.ha.session.ClusterSessionListener"/>
</Cluster>3、將conf/web.xml複製一份到發佈的站點目錄WEB-INF下，在</web-app>前面加入以下這句話<distributable/>
如我的站點目錄爲
[root@essun tomcat]# cd webapps/essun/ [root@essun essun]# ll WEB-INF/ total 168 drwxr-xr-x 2 root root 4096 May 14 04:40 classes drwxr-xr-x 2 root root 4096 May 14 04:40 lib -rw------- 1 root root 162921 May 14 04:40 web.xml
4、在各節點爲使用組播地址添加組播路由，格式：
route add -net $MCAST_ADDRESS netmask 255.255.255.255 dev eth0
如我自己的地址爲
route add -net 228.0.0.4 netmask 255.255.255.255 dev eth0
而在tomcat2上只有兩處與tomcat1不同，其它操作都一樣
jvmRoute="tomcat2"
address="172.16.249.101"
address最好是當前節點的主機地址
session同步到這裏設置完畢。
看一下效果
還有一張
服務器的啓動順序如下:
tomcat1 --> |tomcat2 --> |nginx
先啓動tocmat1,等tomcat1啓動完全的時候再啓動tomcat2，等兩個tocmat全啓動之後，再啓動nginx負載的時候可以用APACHE或NGINX，如果什麼都不用，兩臺一接防問了話，可能會出錯極session不同步的問題，網上也很多人介意不要做seeeion同步，這樣會降低機器的性能，有一個方法可以，就是NGINX的ip_hash,這樣至少可以保證客戶端去防問同一臺TOCMAT，除非那臺TOCMAT掛了