MongoDB實戰系列之六：mongodb的高可用集羣設計實戰

環境：
CentOS 6.0 x64

md01: 10.0.0.11
md02: 10.0.0.12
md03: 10.0.0.14
md04: 10.0.0.15
md05: 10.0.0.16
md06: 10.0.0.17

設計思路：

md01、md02、md03 做一組複製集
md04、md05、md06 做一組複製集
再把這兩組複製集用分片做成 shard1、shard2 用LVS 調用

下載安裝mongodb

cd /root/tools
wget http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.0.0.tgz
tar zxvf mongodb-linux-x86_64-2.0.0.tgz
mv mongodb-linux-x86_64-2.0.0 /elain/apps/mongodb-linux-x86_64-2.0.0
ln -s /elain/apps/mongodb-linux-x86_64-2.0.0 /elain/apps/mongodb
ln -s /elain/apps/mongodb/bin/*  /bin/

#添加用戶組 

/usr/sbin/groupadd -g 690 mongodb
/usr/sbin/useradd -g mongodb mongodb -u 690 -s /sbin/nologin

各節點hosts文件 添加：

true > /etc/hosts
echo -ne "
10.0.0.11     md01
10.0.0.12     md02
10.0.0.14     md03
10.0.0.15     md04
10.0.0.16     md05
10.0.0.17     md06
" >>/etc/hosts

mkdir -p /elain/logs/mongodb/
mkdir -p /elain/data/mongodb/db
/elain/apps/mongodb/bin/mongod --rest --replSet elain/md01:27017 --fork --port 27017 --dbpath /elain/data/mongodb/db/ --logpath /elain/logs/mongodb/mongodb.log

md02

mkdir -p /elain/logs/mongodb/
mkdir -p /elain/data/mongodb/db
/elain/apps/mongodb/bin/mongod --rest --replSet elain/md02:27017 --fork --port 27017 --dbpath /elain/data/mongodb/db/ --logpath /elain/logs/mongodb/mongodb.log

md03

mkdir -p /elain/logs/mongodb/
mkdir -p /elain/data/mongodb/db
/elain/apps/mongodb/bin/mongod --rest --replSet elain/md03:27017 --fork --port 27017 --dbpath /elain/data/mongodb/db/ --logpath /elain/logs/mongodb/mongodb.log

#啓動仲裁節點
在md02上

mkdir /elain/data/mongodb/arb
/elain/apps/mongodb/bin/mongod --fork --rest --replSet elain --dbpath /elain/data/mongodb/arb --port 27015 --logpath /elain/logs/mongodb/mongodb.log

初始化節點:
md01:(登錄其中任何一個節點操作皆可)

> rs.initiate({
    _id : "elain",
    members : [
    {_id : 1, host : "md01:27017"},
    {_id : 2, host : "md02:27017"},
    {_id : 3, host : "md03:27017"},
    {_id : 4, host : "md02:27015", "arbiterOnly": true},
    ]
    });

#驗證

PRIMARY> rs.status()

也可瀏覽：http://10.0.0.11:28017/_replSet  查看狀態

查看副本集狀態

>rs.status()
>user local;
>rs.isMaster()
>db.system.replset.find()

查看當前主庫：

>db.$cmd.findOne({ismaster:1});

另一組副本集同理操作即可;這裏爲省篇幅就不再寫出,但切記兩複製集名稱不可以重複
啓動並配置三臺Config Server

#md01,03,05上執行

mkdir -p /elain/data/mongodb/config/
/elain/apps/mongodb/bin/mongod --configsvr --dbpath /elain/data/mongodb/config/ --port 20000 --logpath /elain/logs/mongodb/config.log --logappend --fork

5、部署並配置三臺Routing Server

指定所有的config sever地址參數，chunkSize是分割數據時每塊(Chunk)的單位大小

#md02,md04,md06

/elain/apps/mongodb/bin/mongos --configdb md01:20000,md03:20000,md05:20000 --port 30000 --chunkSize 100 --logpath /elain/logs/mongodb/mongos.log --logappend --fork

6、命令行添加分片

連接到mongs服務器，並切換到admin

/elain/apps/mongodb/bin/mongo 10.0.0.11:30000/admin

db.runCommand( {
    addshard : "elain/md01:27017",
    name:"shard1",
    maxsize:2048,
    allowLocal:true } );

db.runCommand( {
    addshard : "chujq/md04:27017",
    name:"shard2",
    maxsize:2048,
    allowLocal:true } );

注：添加複製集elain，其中包含一個服務器md01:27017(還餓別的服務器，如md02、md03)，如果md01掛了，mongos會知道它所連接的是一個複製集，並會使用新的主節點（md02或md03）

db.runCommand( { listshards : 1 } );

如果列出(sharding)了以上二個你加的shards，表示shards已經配置成功

導入數據：

mongoimport -d elain -c elain --type csv --headerline --file /root/bak/test.csv --host md02:30000

注：數據自備，我這裏是從生產環境下MYSQL裏導出的一些真實數據來做測試

#激活數據庫分片

db.runCommand( { enablesharding : "elain" } );
mongos> db.runCommand( { enablesharding : "elain" } );
{ "ok" : 1 }

mongos> show dbs
config  0.1875GB
elain   0.453125GB
test    (empty)

#進入數據庫，建立索引，爲分片做準備

/elain/apps/mongodb/bin/mongo 10.0.0.12:30000/admin

use elain;
db.elain.ensureIndex({"client_userid":1});

查看索引：

mongos> db.elain.find({"client_userid" : 151512}).explain();
{
        "cursor" : "BtreeCursor client_userid_1",
        "nscanned" : 3,
        "nscannedObjects" : 3,
        "n" : 3,
        "millis" : 0,
        "nYields" : 0,
        "nChunkSkips" : 0,
        "isMultiKey" : false,
        "indexOnly" : false,
        "indexBounds" : {
                "client_userid" : [
                        [
                                151512,
                                151512
                        ]
                ]
        }
}

#添加分片：

use admin;
db.runCommand( { shardcollection : "elain.elain",key : {client_userid: 1} } )

mongos> db.runCommand( { shardcollection : "elain.elain",key : {client_userid: 1} } )
{ "collectionsharded" : "elain.elain", "ok" : 1 }

#查看分片狀態

use elain;
db.elain.stats();

mongos> db.elain.stats();
{
        "sharded" : true,
        "flags" : 1,
        "ns" : "elain.elain",
        "count" : 507372,
        "numExtents" : 10,
        "size" : 84375328,
        "storageSize" : 97849344,
        "totalIndexSize" : 29253728,
        "indexSizes" : {
                "_id_" : 16474640,
                "client_userid_1" : 12779088
        },
        "avgObjSize" : 166.29874727024747,
        "nindexes" : 2,
        "nchunks" : 1,
        "shards" : {
                "shard2" : {
                        "ns" : "elain.elain",
                        "count" : 507372,
                        "size" : 84375328,
                        "avgObjSize" : 166.29874727024747,
                        "storageSize" : 97849344,
                        "numExtents" : 10,
                        "nindexes" : 2,
                        "lastExtentSize" : 26099712,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 29253728,
                        "indexSizes" : {
                                "_id_" : 16474640,
                                "client_userid_1" : 12779088
                        },
                        "ok" : 1
                }
        },
        "ok" : 1
}

在分片後新寫數據第一次：

mongos> db.elain.stats();
{
        "sharded" : true,
        "flags" : 1,
        "ns" : "elain.elain",
        "count" : 676496,
        "numExtents" : 12,
        "size" : 112500436,
        "storageSize" : 129179648,
        "totalIndexSize" : 47551616,
        "indexSizes" : {
                "_id_" : 21968912,
                "client_userid_1" : 25582704
        },
        "avgObjSize" : 166.29874529930703,
        "nindexes" : 2,
        "nchunks" : 8,
        "shards" : {
                "shard1" : {
                        "ns" : "elain.elain",
                        "count" : 0,
                        "size" : 0,
                        "storageSize" : 8192,
                        "numExtents" : 1,
                        "nindexes" : 2,
                        "lastExtentSize" : 8192,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 16352,
                        "indexSizes" : {
                                "_id_" : 8176,
                                "client_userid_1" : 8176
                        },
                        "ok" : 1
                },
                "shard2" : {
                        "ns" : "elain.elain",
                        "count" : 676496,
                        "size" : 112500436,
                        "avgObjSize" : 166.29874529930703,
                        "storageSize" : 129171456,
                        "numExtents" : 11,
                        "nindexes" : 2,
                        "lastExtentSize" : 31322112,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 47535264,
                        "indexSizes" : {
                                "_id_" : 21960736,
                                "client_userid_1" : 25574528
                        },
                        "ok" : 1
                }
        },
        "ok" : 1
}

分片後新寫數據第二次：

mongos> db.elain.stats();
{
        "sharded" : true,
        "flags" : 1,
        "ns" : "elain.elain",
        "count" : 1189194,
        "numExtents" : 23,
        "size" : 194533928,
        "storageSize" : 252874752,
        "totalIndexSize" : 87262448,
        "indexSizes" : {
                "_id_" : 43692544,
                "client_userid_1" : 43569904
        },
        "avgObjSize" : 163.58468677103988,
        "nindexes" : 2,
        "nchunks" : 8,
        "shards" : {
                "shard1" : {
                        "ns" : "elain.elain",
                        "count" : 396370,
                        "size" : 62195348,
                        "avgObjSize" : 156.91234957236924,
                        "storageSize" : 86114304,
                        "numExtents" : 11,
                        "nindexes" : 2,
                        "lastExtentSize" : 20881408,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 35949872,
                        "indexSizes" : {
                                "_id_" : 17954496,
                                "client_userid_1" : 17995376
                        },
                        "ok" : 1
                },
                "shard2" : {
                        "ns" : "elain.elain",
                        "count" : 792824,
                        "size" : 132338580,
                        "avgObjSize" : 166.9205019020615,
                        "storageSize" : 166760448,
                        "numExtents" : 12,
                        "nindexes" : 2,
                        "lastExtentSize" : 37588992,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 51312576,
                        "indexSizes" : {
                                "_id_" : 25738048,
                                "client_userid_1" : 25574528
                        },
                        "ok" : 1
                }
        },
        "ok" : 1
}

分片後新寫數據第三次：

mongos> db.elain.stats();
{
        "sharded" : true,
        "flags" : 1,
        "ns" : "elain.elain",
        "count" : 1376876,
        "numExtents" : 23,
        "size" : 225576604,
        "storageSize" : 252874752,
        "totalIndexSize" : 100826432,
        "indexSizes" : {
                "_id_" : 50249696,
                "client_userid_1" : 50576736
        },
        "avgObjSize" : 163.83218532387812,
        "nindexes" : 2,
        "nchunks" : 10,
        "shards" : {
                "shard1" : {
                        "ns" : "elain.elain",
                        "count" : 494202,
                        "size" : 77551984,
                        "avgObjSize" : 156.92365469990003,
                        "storageSize" : 86114304,
                        "numExtents" : 11,
                        "nindexes" : 2,
                        "lastExtentSize" : 20881408,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 42057344,
                        "indexSizes" : {
                                "_id_" : 21600992,
                                "client_userid_1" : 20456352
                        },
                        "ok" : 1
                },
                "shard2" : {
                        "ns" : "elain.elain",
                        "count" : 882674,
                        "size" : 148024620,
                        "avgObjSize" : 167.70021548159343,
                        "storageSize" : 166760448,
                        "numExtents" : 12,
                        "nindexes" : 2,
                        "lastExtentSize" : 37588992,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 58769088,
                        "indexSizes" : {
                                "_id_" : 28648704,
                                "client_userid_1" : 30120384
                        },
                        "ok" : 1
                }
        },
        "ok" : 1
}

分片後新寫數據第四次（DOWN 掉md04的mongo服務）

mongos> db.elain.stats();
{
        "sharded" : true,
        "flags" : 1,
        "ns" : "elain.elain",
        "count" : 1686310,
        "numExtents" : 26,
        "size" : 275761376,
        "storageSize" : 353116160,
        "totalIndexSize" : 129033632,
        "indexSizes" : {
                "_id_" : 63265888,
                "client_userid_1" : 65767744
        },
        "avgObjSize" : 163.52946729842083,
        "nindexes" : 2,
        "nchunks" : 10,
        "shards" : {
                "shard1" : {
                        "ns" : "elain.elain",
                        "count" : 740264,
                        "size" : 116213588,
                        "avgObjSize" : 156.98938216636228,
                        "storageSize" : 141246464,
                        "numExtents" : 13,
                        "nindexes" : 2,
                        "lastExtentSize" : 30072832,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 61810560,
                        "indexSizes" : {
                                "_id_" : 32556832,
                                "client_userid_1" : 29253728
                        },
                        "ok" : 1
                },
                "shard2" : {
                        "ns" : "elain.elain",
                        "count" : 946046,
                        "size" : 159547788,
                        "avgObjSize" : 168.64696642657967,
                        "storageSize" : 211869696,
                        "numExtents" : 13,
                        "nindexes" : 2,
                        "lastExtentSize" : 45109248,
                        "paddingFactor" : 1,
                        "flags" : 1,
                        "totalIndexSize" : 67223072,
                        "indexSizes" : {
                                "_id_" : 30709056,
                                "client_userid_1" : 36514016
                        },
                        "ok" : 1
                }
        },
        "ok" : 1
}

總結：通過以上四次的寫數據測試，我們可以看到分片是成功的，每次寫數據,shard1、shard2都有數據寫入，且，在下面的複製集中DOWN 掉任意一臺，不影響整個架構的正常服務。

刪除片操作

mongos> db.runCommand({"removeshard" : "10.0.0.11:27011"});
{
        "msg" : "draining started successfully",
        "state" : "started",
        "shard" : "shard2",
        "ok" : 1
}

再執行，可看到removeshard的挪動進度

mongos> db.runCommand({"removeshard" : "10.0.0.11:27011"});
{
        "msg" : "draining ongoing",
        "state" : "ongoing",
        "remaining" : {
                "chunks" : NumberLong(3),
                "dbs" : NumberLong(0)
        },
        "ok" : 1
}

轉載請註明： 轉載自http://www.elain.org

本文鏈接地址:MongoDB實戰系列之六：mongodb的高可用集羣設計實戰