接續接上篇 緩存時代來臨 爲藍本,繼續改造我們的百萬級站點架構,這次我們
拿之前存儲靜態內容的 nfs 開刀,衆所周知 nfs 的多臺集羣節點下可能由於多重
原因(磁盤io , 網絡帶寬, 併發場景),不適合做文件共享系統的基礎結構.
互聯網站點中,存在大量圖片或其他靜態內容,並且這些內容一般在1M之內,對於
海量小文件,我們將採用mogilefs分佈式文件系統來完成.其中概念自行google.
# mogilefs分佈式文件系統工作流程
架構已經愈發複雜,我們需要從新梳理一下.從下表中應該很容易看出第三彈的
任務,重新規劃如下:
# 羅馬數字表示在何階段規劃,(*)代表已廢棄
slave1.king.com
172.16.43.1
DNS輪詢 -> slave1.king.com , slave2.king.com(I)
haproxy七層代理流量分離 -> imgs , text , dynamic(I)
keepalived 爲haproxy HA(I)
slave2.king.com
172.16.43.2
haproxy七層代理流量分離 -> imgs , text , dynamic(I)
keepalived 爲haproxy HA(I)
slave3.king.com
172.16.43.3
nginx虛擬主機組 -> imgs1.king.com , imgs2.king.com
text1.king.com , text2.king.com
dynamic1.king.com(I)
php-fpm模塊(I)
mysql數據庫(I)
nfs /nfsshared(*) New!!
varnish imgs 緩存(II)
memcache session, key/value cache(II)
mogilefs tracker node(III) New!!
mogilefs store node(III) New!!
slave4.king.com
172.16.43.4
nginx虛擬主機 -> dynamic2.king.com(I)
php-fpm模塊(I)
mysql-proxy(I)
mysql數據庫(I)
varnish text 緩存(II)
memcache session, key/value cache(II)
mogilefs tracker node(III) New!!
mogilefs store node(III) New!!
# 新規劃內容如下
操作過程
1. 安裝配置階段(slave3.king.com , slave4.king.com) # 準備如下包,因爲我們要安裝tracker節點和store節點 MogileFS-Server-2.46-2.el6.noarch.rpm MogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm MogileFS-Server-mogstored-2.46-2.el6.noarch.rpm MogileFS-Utils-2.19-1.el6.noarch.rpm perl-MogileFS-Client-1.14-1.el6.noarch.rpm perl-Net-Netmask-1.9015-8.el6.noarch.rpm perl-Perlbal-1.78-1.el6.noarch.rpm # # 安裝mogilefs以及創建store目錄,改變權限 yum -y install *.rpm perl-IO-AIO -y mkdir /dfs/mogdata/dev1 -pv (slace4上建dev2) chown -R mogilefs.mogilefs /var/run/mogilefsd /dfs/mogdata # # mysql -e 授權 建庫(僅slave3執行) mysql -e "create database mogilefs;" mysql -e "grant all on *.* to 'root'@'172.16.%.%' identified by '';" mysql -e "grant all on mogilefs.* 'moguser'@'172.16.%.%' identified by 'mogpass';" mysql -e "flush privileges;" # # 初始化mogilefs所需數據庫(僅slave3執行) mogdbsetup --dbhost=172.16.43.3 --dbport=3306 --dbname=mogilefs --dbuser=moguser --dbpass=mogpass --yes
# 接 1. 安裝配置階段(slave3.king.com , slave4.king.com) 操作 # mogilefs配置文件 vim /etc/mogilefs/mogilefsd.conf # Enable daemon mode to work in background and use syslog daemonize = 1 # Where to store the pid of the daemon (must be the same in the init script) pidfile = /var/run/mogilefsd/mogilefsd.pid # Database connection information db_dsn = DBI:mysql:mogilefs:host=172.16.43.3 db_user = moguser db_pass = mogpass # IP:PORT to listen on for mogilefs client requests listen = 172.16.43.3:7001 # Optional, if you don't define the port above. conf_port = 7001 # Number of query workers to start by default. query_jobs = 10 # Number of delete workers to start by default. delete_jobs = 1 # Number of replicate workers to start by default. replicate_jobs = 5 # Number of reaper workers to start by default. # (you don't usually need to increase this)# reaper_jobs = 1 # # store配置文件 vim /etc/mogilefs/mogstored.conf maxconns = 10000 httplisten = 0.0.0.0:7500 mgmtlisten = 0.0.0.0:7501 docroot = /dfs/mogdata # # 分別啓動tracker以及store服務 (觀察7001與7500,7501端口) service mogilefsd start service mogstored start
# 觀察服務啓動情況
2. 添加分佈式文件系統(slave3.king.com) # 添加分佈式節點信息 mogadm --tracker=172.16.43.3:7001 host add 172.16.43.3 --ip=172.16.43.3 --status=alive mogadm --tracker=172.16.43.4:7001 host add 172.16.43.4 --ip=172.16.43.4 --status=alive mogadm --tracker=172.16.43.3:7001 host list # # 添加分佈式設備節點信息 mogadm --tracker=172.16.43.3:7001 device add 172.16.43.3 1 mogadm --tracker=172.16.43.4:7001 device add 172.16.43.4 2 mogadm --tracker=172.16.43.3:7001 device list # # 添加分佈式域節點信息 mogadm --tracker=172.16.43.3:7001 domain add images mogadm --tracker=172.16.43.3:7001 domain add text mogadm --tracker=172.16.43.3:7001 domain list
# mogilefs 所需 host , device , domain 添加準備就緒情況
3. ngxin反向代理解析文件
# 將第三方模塊編譯進nginx,先將之前的nginx服務停止,防止意外
service nginx stop
cd nginx-1.4.7
./configure --prefix=/usr/local/nginx --sbin-path=/usr/local/nginx/nginx --conf-path=/etc/nginx/nginx.conf --pid-path=/usr/local/nginx/nginx.pid --with-pcre --add-module=/root/mogilefs/nginx-mogilefs-module-master
make && make install
#
nginx-mogilefs-module-master.zip
#
http {
upstream mogcluster {
server 172.16.43.3:7001;
server 172.16.43.4:7001;
}
#
# 之前的系統圖片設置到images與imgs考慮程序一致性,這裏
location /images/ {
mogilefs_tracker mogcluster;
mogilefs_domain images;
mogilefs_methods GET PUT DELETE;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
#
location /imgs/ {
mogilefs_tracker mogcluster;
mogilefs_domain images;
mogilefs_methods GET PUT DELETE;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
#
# 之前系統靜態文件分爲html,css,js等目錄,所以這裏需要目錄定位
location ~* ^(/(static|css|js)/.*)$ {
mogilefs_tracker mogcluster;
mogilefs_domain text;
mogilefs_pass $1 {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
}4. 以上一篇架構爲例,添加測試文件 mogupload --trackers=172.16.43.3:7001 --domain=images --key='/2.jpg' --file='./imgs/2.jpg' mogupload --trackers=172.16.43.3:7001 --domain=images --key='/1.jpg' --file='./images/1.jpg' mogupload --trackers=172.16.43.3:7001 --domain=text --key='static/index.html' --file='./static/index.html' mogupload --trackers=172.16.43.3:7001 --domain=text --key='css/test.css' --file='./css/test.css' mogupload --trackers=172.16.43.3:7001 --domain=text --key='js/test.js' --file='./js/test.js'
# 測試
service nginx restart
service varnish restart
使用瀏覽器隱身模式訪問
最後附完整slave3.king.com中nginx的配置文件
#user nobody;
worker_processes 1;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 5;
gzip on;
upstream mogcluster {
server 172.16.43.3:7001;
server 172.16.43.4:7001;
}
server {
listen 80;
server_name dynamic1.king.com;
access_log /var/log/nginx/dynamic1.access.log;
location ~ \.php$ {
root /nfsshared/html;
fastcgi_pass 172.16.43.3:9000;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
}
location / {
root /nfsshared/html;
index index.php index.html index.htm;
}
}
server {
listen 80;
server_name imgs1.king.com;
access_log /var/log/nginx/imgs1.access.log;
location /images/ {
mogilefs_tracker mogcluster;
mogilefs_domain images;
mogilefs_methods GET PUT DELETE;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location /imgs/ {
mogilefs_tracker mogcluster;
mogilefs_domain images;
mogilefs_methods GET PUT DELETE;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
error_page 404 /404.html;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
server {
listen 80;
server_name imgs2.king.com;
access_log /var/log/nginx/imgs2.access.log;
location /images/ {
mogilefs_tracker mogcluster;
mogilefs_domain images;
mogilefs_methods GET PUT DELETE;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location /imgs/ {
mogilefs_tracker mogcluster;
mogilefs_domain images;
mogilefs_methods GET PUT DELETE;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
error_page 404 /404.html;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
server {
listen 80;
server_name text1.king.com;
access_log /var/log/nginx/text1.access.log;
location ~* ^(/(static|css|js)/.*)$ {
mogilefs_tracker mogcluster;
mogilefs_domain text;
mogilefs_pass $1 {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location / {
index index.html index.htm;
}
error_page 404 /404.html;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
server {
listen 80;
server_name text2.king.com;
access_log /var/log/nginx/text2.access.log;
location ~* ^(/(static|css|js)/.*)$ {
mogilefs_tracker mogcluster;
mogilefs_domain text;
mogilefs_pass $1 {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location / {
index index.html index.htm;
}
error_page 404 /404.html;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}