MongoDB健壮集群——用副本集做分片
1. MongoDB分片+副本集
健壮的集群方案
多个配置服务器 多个mongos服务器 每个片都是副本集 正确设置w
架构图
|
说明： 1. 此实验环境在一台机器上通过不同port和dbpath实现启动不同的mongod实例 2. 总的9个mongod实例，分别做成shard1、shard2、shard3三组副本集，每组1主2从 3. Mongos进程的数量不限，建议把mongos配置在每个应用服务器本机上，这样每个应用服务器就与自身的mongos进行通信，如果服务器不工作了，并不会影响其他的应用服务器与其自己的mongos通信 4. 此实验模拟2台应用服务器（2个mongos服务） 5. 生产环境中每个片都应该是副本集，这样单个服务器坏了，才不会导致片失效 |
部署环境
创建相关目录
[root@Master cluster2]# mkdir -p shard{1,2,3}/node{1,2,3} [root@Master cluster2]# mkdir -p shard{1,2,3}/logs [root@Master cluster2]# ls shard* shard1: logs node1 node2 node3 shard2: logs node1 node2 node3 shard3: logs node1 node2 node3 [root@Master cluster2]# mkdir -p config/logs [root@Master cluster2]# mkdir -p config/node{1,2,3} [root@Master cluster2]# ls config/ logs node1 node2 node3 [root@Master cluster2]# mkdir -p mongos/logs
启动配置服务
|
Config server |
/data/mongodb/config/node1 |
/data/mongodb/config/logs/node1.log |
10000 |
|
/data/mongodb/config/node2 |
/data/mongodb/config/logs/node2.log |
20000 |
|
|
/data/mongodb/config/node3 |
/data/mongodb/config/logs/node3.log |
30000 |
#按规划启动3个：跟启动单个配置服务一样，只是重复3次
[root@Master cluster2]# mongod --dbpath config/node1 --logpath config/logs/node1.log --logappend --fork --port 10000 [root@Master cluster2]# mongod --dbpath config/node2 --logpath config/logs/node2.log --logappend --fork --port 20000 [root@Master cluster2]# mongod --dbpath config/node3 --logpath config/logs/node3.log --logappend --fork --port 30000 [root@Master cluster2]# ps -ef|grep mongod|grep -v grep mongod 2329 1 0 20:05 ? 00:00:02 /usr/bin/mongod -f /etc/mongod.conf root 2703 1 1 20:13 ? 00:00:00 mongod --dbpath config/node1 --logpath config/logs/node1.log --logappend --fork --port 10000 root 2716 1 1 20:13 ? 00:00:00 mongod --dbpath config/node2 --logpath config/logs/node2.log --logappend --fork --port 20000 root 2729 1 1 20:13 ? 00:00:00 mongod --dbpath config/node3 --logpath config/logs/node3.log --logappend --fork --port 30000
启动路由服务
|
Mongos server |
—— |
/data/mongodb/mongos/logs/node1.log |
40000 |
|
—— |
/data/mongodb/mongos/logs/node2.log |
50000 |
#mongos的数量不受限制，通常应用一个服务器运行一个mongos
[root@Master cluster2]# mongos --port 40000 --configdb localhost:10000,localhost:20000,localhost:30000 --logpath mongos/logs/mongos1.log --logappend --fork [root@Master cluster2]# mongos --port 50000 --configdb localhost:10000,localhost:20000,localhost:30000 --logpath mongos/logs/mongos1.log --logappend --fork [root@Master cluster2]# ps -ef|grep mongos|grep -v grep root 2809 1 0 20:18 ? 00:00:00 mongos --port 40000 --configdb localhost:10000,localhost:20000,localhost:30000 --logpath mongos/logs/mongos1.log --logappend --fork root 2862 1 0 20:19 ? 00:00:00 mongos --port 50000 --configdb localhost:10000,localhost:20000,localhost:30000 --logpath mongos/logs/mongos1.log --logappend --fork
配置副本集
按规划，配置启动shard1、shard2、shard3三组副本集
#此处以shard1为例说明配置方法
#启动三个mongod进程
[root@Master cluster2]# mongod --replSet shard1 --dbpath shard1/node1 --logpath shard1/logs/node1.log --logappend --fork --port 10001 [root@Master cluster2]# mongod --replSet shard1 --dbpath shard1/node2 --logpath shard1/logs/node2.log --logappend --fork --port 10002 [root@Master cluster2]# mongod --replSet shard1 --dbpath shard1/node3 --logpath shard1/logs/node3.log --logappend --fork --port 10003
#初始化Replica Set:shard1
[root@Master cluster2]# mongo --port 10001 MongoDB shell version: 3.0.5 connecting to: 127.0.0.1:10001/test > use admin switched to db admin > rsconf={"_id" : "shard1","members" : [{"_id" : 0, "host" : "localhost:10001"}]} { "_id" : "shard1", "members" : [ { "_id" : 0, "host" : "localhost:10001" } ] } > rs.initiate(rsconf) { "ok" : 1 } shard1:OTHER> rs.add("localhost:10002") { "ok" : 1 } shard1:PRIMARY> rs.add("localhost:10003") { "ok" : 1 } shard1:PRIMARY> rs.conf() { "_id" : "shard1", "version" : 3, "members" : [ { "_id" : 0, "host" : "localhost:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "localhost:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "localhost:10003", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
Shard2和shard3同shard1配置副本集
[root@Master cluster2]# mongod --replSet shard2 --dbpath shard2/node1 --logpath shard2/logs/node1.log --logappend --fork --port 20001 [root@Master cluster2]# mongod --replSet shard2 --dbpath shard2/node2 --logpath shard2/logs/node2.log --logappend --fork --port 20002 [root@Master cluster2]# mongod --replSet shard2 --dbpath shard2/node3 --logpath shard2/logs/node3.log --logappend --fork --port 20003 [root@Master cluster2]# mongo --port 20001 > use admin > rsconf={"_id" : "shard2","members" : [{"_id" : 0, "host" : "localhost:20001"}]} > rs.initiate(rsconf) shard2:OTHER> rs.add("localhost:20002") shard2:PRIMARY> rs.add("localhost:20003") shard2:PRIMARY> rs.conf() { "_id" : "shard2", "version" : 3, "members" : [ { "_id" : 0, "host" : "localhost:20001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "localhost:20002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "localhost:20003", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
[root@Master cluster2]# mongod --replSet shard3 --dbpath shard3/node1 --logpath shard3/logs/node1.log --logappend --fork --port 30001 [root@Master cluster2]# mongod --replSet shard3 --dbpath shard3/node2 --logpath shard3/logs/node2.log --logappend --fork --port 30002 [root@Master cluster2]# mongod --replSet shard3 --dbpath shard3/node3 --logpath shard3/logs/node3.log --logappend --fork --port 30003 [root@Master cluster2]# mongo --port 30001 connecting to: 127.0.0.1:30001/test> use admin > rsconf={"_id" : "shard3","members" : [{"_id" : 0, "host" : "localhost:30001"}]}
> rs.initiate(rsconf) shard3:OTHER> rs.add("localhost:30002") shard3:PRIMARY> rs.add("localhost:30003") shard3:PRIMARY> rs.conf() { "_id" : "shard3", "version" : 3, "members" : [ { "_id" : 0, "host" : "localhost:30001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "localhost:30002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "localhost:30003", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
添加（副本集）分片
#连接到mongs，并切换到admin这里必须连接路由节点
[root@Master cluster2]# mongo --port 40000 MongoDB shell version: 3.0.5 connecting to: 127.0.0.1:40000/test mongos> use admin switched to db admin mongos> db.runCommand({"addShard":"shard1/localhost:10001"}) { "shardAdded" : "shard1", "ok" : 1 } mongos> db.runCommand({"addShard":"shard2/localhost:20001"}) { "shardAdded" : "shard2", "ok" : 1 } mongos> db.runCommand({"addShard":"shard3/localhost:30001"}) { "shardAdded" : "shard3", "ok" : 1 } mongos> db.runCommand({listshards:1}) { "shards" : [ { "_id" : "shard1", "host" : "shard1/localhost:10001,localhost:10002,localhost:10003" }, { "_id" : "shard2", "host" : "shard2/localhost:20001,localhost:20002,localhost:20003" }, { "_id" : "shard3", "host" : "shard3/localhost:30001,localhost:30002,localhost:30003" } ], "ok" : 1 }
激活db和collections分片
激活数据库分片，命令
> db.runCommand( { enablesharding : "数据库名称"} );
执行以上命令，可以让数据库跨shard，如果不执行这步，数据库只会存放在一个shard
一旦激活数据库分片，数据库中不同的collection将被存放在不同的shard上
但一个collection仍旧存放在同一个shard上，要使单个collection也分片，还需单独对collection作些操作
#如：激活test数据库分片功能，连接mongos进程
[root@Master cluster2]# mongo --port 50000 MongoDB shell version: 3.0.5 connecting to: 127.0.0.1:50000/test mongos> use admin switched to db admin mongos> db.runCommand({"enablesharding":"test"}) { "ok" : 1 }
要使单个collection也分片存储，需要给collection指定一个分片key，通过以下命令操作：
> db.runCommand( { shardcollection : "集合名称",key : "字段名称"});
注： a. 分片的collection系统会自动创建一个索引（也可用户提前创建好）
b. 分片的collection只能有一个在分片key上的唯一索引，其它唯一索引不被允许
#对collection：test.yujx分片
mongos> db.runCommand({"shardcollection":"test.yujx","key":{"_id":1}})
{ "collectionsharded" : "test.yujx", "ok" : 1 }
生成测试数据
mongos> use test switched to db test mongos> for(var i=1;i<=88888;i++) db.yujx.save({"id":i,"a":123456789,"b":888888888,"c":100000000}) WriteResult({ "nInserted" : 1 }) mongos> db.yujx.count() 88888
查看集合分片
mongos> db.yujx.stats() { "sharded" : true, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "ns" : "test.yujx", "count" : 88888, "numExtents" : 9, "size" : 9955456, "storageSize" : 22523904, "totalIndexSize" : 2918832, "indexSizes" : { "_id_" : 2918832 }, "avgObjSize" : 112, "nindexes" : 1, "nchunks" : 3, "shards" : { "shard1" : { "ns" : "test.yujx", "count" : 8, "size" : 896, "avgObjSize" : 112, "numExtents" : 1, "storageSize" : 8192, "lastExtentSize" : 8192, "paddingFactor" : 1, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "nindexes" : 1, "totalIndexSize" : 8176, "indexSizes" : { "_id_" : 8176 }, "ok" : 1, "$gleStats" : { "lastOpTime" : Timestamp(0, 0), "electionId" : ObjectId("55d15366716d7504d5d74c4c") } }, "shard2" : { "ns" : "test.yujx", "count" : 88879, "size" : 9954448, "avgObjSize" : 112, "numExtents" : 7, "storageSize" : 22507520, "lastExtentSize" : 11325440, "paddingFactor" : 1, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "nindexes" : 1, "totalIndexSize" : 2902480, "indexSizes" : { "_id_" : 2902480 }, "ok" : 1, "$gleStats" : { "lastOpTime" : Timestamp(0, 0), "electionId" : ObjectId("55d1543eabed7d6d4a71d25e") } }, "shard3" : { "ns" : "test.yujx", "count" : 1, "size" : 112, "avgObjSize" : 112, "numExtents" : 1, "storageSize" : 8192, "lastExtentSize" : 8192, "paddingFactor" : 1, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "nindexes" : 1, "totalIndexSize" : 8176, "indexSizes" : { "_id_" : 8176 }, "ok" : 1, "$gleStats" : { "lastOpTime" : Timestamp(0, 0), "electionId" : ObjectId("55d155346f36550e3c5f062c") } } }, "ok" : 1 }
查看数据库分片
mongos> db.printShardingStatus() --- Sharding Status --- sharding version: { "_id" : 1, "minCompatibleVersion" : 5, "currentVersion" : 6, "clusterId" : ObjectId("55d152a35348652fbc726a10") } shards: { "_id" : "shard1", "host" : "shard1/localhost:10001,localhost:10002,localhost:10003" } { "_id" : "shard2", "host" : "shard2/localhost:20001,localhost:20002,localhost:20003" } { "_id" : "shard3", "host" : "shard3/localhost:30001,localhost:30002,localhost:30003" } balancer: Currently enabled: yes Currently running: yes Balancer lock taken at Sun Aug 16 2015 20:43:49 GMT-0700 (PDT) by Master.Hadoop:50000:1439781543:1804289383:Balancer:846930886 Failed balancer rounds in last 5 attempts: 0 Migration Results for the last 24 hours: 2 : Success 1 : Failed with error 'could not acquire collection lock for test.yujx to migrate chunk [{ : MinKey },{ : MaxKey }) :: caused by :: Lock for migrating chunk [{ : MinKey }, { : MaxKey }) in test.yujx is taken.', from shard1 to shard2 databases: { "_id" : "admin", "partitioned" : false, "primary" : "config" } { "_id" : "test", "partitioned" : true, "primary" : "shard1" } test.yujx shard key: { "_id" : 1 } chunks: shard1 1 shard2 1 shard3 1 { "_id" : { "$minKey" : 1 } } -->> { "_id" : ObjectId("55d157cca0c90140e33a9342") } on : shard3 Timestamp(3, 0) { "_id" : ObjectId("55d157cca0c90140e33a9342") } -->> { "_id" : ObjectId("55d157cca0c90140e33a934a") } on : shard1 Timestamp(3, 1) { "_id" : ObjectId("55d157cca0c90140e33a934a") } -->> { "_id" : { "$maxKey" : 1 } } on : shard2 Timestamp(2, 0)
#或者连接mongos的config数据库查询
mongos> use config switched to db config mongos> db.shards.find() { "_id" : "shard1", "host" : "shard1/localhost:10001,localhost:10002,localhost:10003" } { "_id" : "shard2", "host" : "shard2/localhost:20001,localhost:20002,localhost:20003" } { "_id" : "shard3", "host" : "shard3/localhost:30001,localhost:30002,localhost:30003" } mongos> db.databases.find() { "_id" : "admin", "partitioned" : false, "primary" : "config" } { "_id" : "test", "partitioned" : true, "primary" : "shard1" } mongos> db.chunks.find() { "_id" : "test.yujx-_id_MinKey", "lastmod" : Timestamp(3, 0), "lastmodEpoch" : ObjectId("55d15738679c4d5f9108eba0"), "ns" : "test.yujx", "min" : { "_id" : { "$minKey" : 1 } }, "max" : { "_id" : ObjectId("55d157cca0c90140e33a9342") }, "shard" : "shard3" } { "_id" : "test.yujx-_id_ObjectId('55d157cca0c90140e33a9342')", "lastmod" : Timestamp(3, 1), "lastmodEpoch" : ObjectId("55d15738679c4d5f9108eba0"), "ns" : "test.yujx", "min" : { "_id" : ObjectId("55d157cca0c90140e33a9342") }, "max" : { "_id" : ObjectId("55d157cca0c90140e33a934a") }, "shard" : "shard1" } { "_id" : "test.yujx-_id_ObjectId('55d157cca0c90140e33a934a')", "lastmod" : Timestamp(2, 0), "lastmodEpoch" : ObjectId("55d15738679c4d5f9108eba0"), "ns" : "test.yujx", "min" : { "_id" : ObjectId("55d157cca0c90140e33a934a") }, "max" : { "_id" : { "$maxKey" : 1 } }, "shard" : "shard2" }
hash分片
MongoDB2.4以上的版本支持基于哈希的分片
mongos> use admin mongos> db.runCommand({"enablesharding":"mydb"}) mongos> db.runCommand({"shardcollection":"mydb.mycollection","key":{"_id":"hashed"}}) mongos> use mydb switched to db mydb mongos> for(i=0;i<8888;i++){ db.mycollection.insert({"Uid":i,"Name":"zhanjindong2","Age":13,"Date":new Date()}); } WriteResult({ "nInserted" : 1 }) mongos> db.mycollection.stats() { "sharded" : true, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "ns" : "mydb.mycollection", "count" : 8888, "numExtents" : 12, "size" : 995456, "storageSize" : 2088960, "totalIndexSize" : 735840, "indexSizes" : { "_id_" : 318864, "_id_hashed" : 416976 }, "avgObjSize" : 112, "nindexes" : 2, "nchunks" : 6, "shards" : { "shard1" : { "ns" : "mydb.mycollection", "count" : 2975, "size" : 333200, "avgObjSize" : 112, "numExtents" : 4, "storageSize" : 696320, "lastExtentSize" : 524288, "paddingFactor" : 1, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "nindexes" : 2, "totalIndexSize" : 245280, "indexSizes" : { "_id_" : 106288, "_id_hashed" : 138992 }, "ok" : 1, "$gleStats" : { "lastOpTime" : Timestamp(0, 0), "electionId" : ObjectId("55d15366716d7504d5d74c4c") } }, "shard2" : { "ns" : "mydb.mycollection", "count" : 2909, "size" : 325808, "avgObjSize" : 112, "numExtents" : 4, "storageSize" : 696320, "lastExtentSize" : 524288, "paddingFactor" : 1, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "nindexes" : 2, "totalIndexSize" : 245280, "indexSizes" : { "_id_" : 106288, "_id_hashed" : 138992 }, "ok" : 1, "$gleStats" : { "lastOpTime" : Timestamp(0, 0), "electionId" : ObjectId("55d1543eabed7d6d4a71d25e") } }, "shard3" : { "ns" : "mydb.mycollection", "count" : 3004, "size" : 336448, "avgObjSize" : 112, "numExtents" : 4, "storageSize" : 696320, "lastExtentSize" : 524288, "paddingFactor" : 1, "paddingFactorNote" : "paddingFactor is unused and unmaintained in 3.0. It remains hard coded to 1.0 for compatibility only.", "userFlags" : 1, "capped" : false, "nindexes" : 2, "totalIndexSize" : 245280, "indexSizes" : { "_id_" : 106288, "_id_hashed" : 138992 }, "ok" : 1, "$gleStats" : { "lastOpTime" : Timestamp(0, 0), "electionId" : ObjectId("55d155346f36550e3c5f062c") } } }, "ok" : 1 }
单点故障分析
由于这是为了了解入门mongodb做的实验，而故障模拟太浪费时间，所以这里就不一一列出，关于故障场景分析，可以参考：
http://blog.itpub.net/27000195/viewspace-1404402/
