mongodb同步ElasticSearch性能测试脚本记录
一、逐条写入性能测试脚本
[root@wlan-cloudserver8 bin]# ./mongo 172.27.8.118:40000
SHARDING_2:PRIMARY> use admin
switched to db admin
SHARDING_2:PRIMARY> db.auth('wlan','wlan'）
SHARDING_2:PRIMARY> use estestdbnew
switched to db estestdbnew
SHARDING_2:PRIMARY> for(var i=0;i<1000;i++){db.estestdbnew.insert({name:"new", age:999})}
二、通过命令行批量写入性能测试脚本
var bulk = db.test.initializeUnorderedBulkOp();
for(var i=0;i<100000;i++){bulk.insert( { item: "abc123", defaultQty: 100, status: "A", points: 100 } )} ---写入十万条
bulk.execute();
三、通过代码批量写入性能测试脚本
package mongodbtest; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; public class BulkWriting { public static void main(String[] args) throws IOException, ParseException { MongoDBConnector mongoDBConnector = new MongoDBConnector("admin", "admin", "estestdbnew", "172.27.8.110", 27017, "test"); mongoDBConnector.connect(); // mongoDBConnector.insertone(); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); sdf.setTimeZone(TimeZone.getTimeZone("GMT+8")); System.out.println("start:" + sdf.format(new Date())); mongoDBConnector.bulkwirte(); System.out.println("finish:" + sdf.format(new Date())); } }
package mongodbtest; import java.util.ArrayList; import java.util.List; import org.bson.Document; import org.bson.conversions.Bson; import com.mongodb.MongoClient; import com.mongodb.MongoCredential; import com.mongodb.ReadPreference; import com.mongodb.ServerAddress; import com.mongodb.bulk.BulkWriteResult; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.InsertOneModel; import com.mongodb.client.model.WriteModel; public class MongoDBConnector { private String mongodbUserName; private String mongodbPassWord; private String mongodbDBName; private String mongodbServerAddr; private int mongodbServerport; private String mongodbCollection; private MongoClient mongoClient; private MongoCollection<Document> collection; public MongoDBConnector(String mongodbUserName, String mongodbPassWord, String mongodbDBName, String mongodbServerAddr, int mongodbServerport, String mongodbCollection) { this.mongodbUserName = mongodbUserName; this.mongodbPassWord = mongodbPassWord; this.mongodbDBName = mongodbDBName; this.mongodbServerAddr = mongodbServerAddr; this.mongodbServerport = mongodbServerport; this.mongodbCollection = mongodbCollection; } public void connect() { ServerAddress serverAddress = new ServerAddress(this.mongodbServerAddr, this.mongodbServerport); List<ServerAddress> addrs = new ArrayList<ServerAddress>(); addrs.add(serverAddress); MongoCredential credential = MongoCredential.createScramSha1Credential(this.mongodbUserName, this.mongodbDBName, this.mongodbPassWord.toCharArray()); List<MongoCredential> credentials = new ArrayList<MongoCredential>(); credentials.add(credential); MongoClient mongoClient = new MongoClient(addrs, credentials); this.mongoClient = mongoClient; MongoDatabase mongoDatabase = mongoClient.getDatabase(this.mongodbDBName) .withReadPreference(ReadPreference.secondary()); MongoCollection<Document> collection = mongoDatabase.getCollection(this.mongodbCollection); this.collection = collection; return; } public MongoCursor<Document> find(Bson filter) { FindIterable<Document> findIterable = this.collection.find(filter);// .batchSize(100000); MongoCursor<Document> mongoCursor = findIterable.iterator(); return mongoCursor; } public void insertone() { Document document = new Document("x", 1); collection.insertOne(document); document.append("x", 2).append("y", 3); System.out.println("insert one document has finished!"); } public void bulkwirte() { List<WriteModel<Document>> writes = new ArrayList<WriteModel<Document>>(); for (int i = 100000; i < 300000; i++) { writes.add(new InsertOneModel<Document>( new Document("_id", i).append("char", "Taeln").append("class", "fighter").append("lvl", 4))); } BulkWriteResult bulkWriteResult = collection.bulkWrite(writes); System.out.println(bulkWriteResult); } public void close() { this.mongoClient.close(); return; } }
测试结果：
start:2016-11-22 11:49:50.771 AcknowledgedBulkWriteResult{insertedCount=200000, matchedCount=0, removedCount=0, modifiedCount=0, upserts=[]} finish:2016-11-22 11:50:25.201
观察ES，大概在mongodb写完3分钟后才完全同步过去
[root@master ~]# curl '172.27.8.132:9200/_cat/indices?v' health status index pri rep docs.count docs.deleted store.size pri.store.size green open estestdb 5 1 27826 0 11.6mb 5.8mb green open logstash-2015.05.20 5 1 4750 0 35.9mb 17.9mb green open mongodb_meta 5 1 327826 0 56.2mb 28.1mb green open .marvel-es-1-2016.11.22 1 1 34554 96 33.8mb 16.9mb green open .marvel-es-1-2016.11.21 1 1 211946 564 203.8mb 101.8mb green open .marvel-es-1-2016.11.20 1 1 204352 740 201.5mb 100.7mb green open bank 5 1 1000 0 885.2kb 442.6kb green open .kibana 1 1 5 0 51.7kb 25.8kb green open .marvel-es-data-1 1 1 20 4 7.4mb 3.7mb green open .marvel-es-1-2016.11.19 1 1 195921 728 191.4mb 95.7mb green open .marvel-es-1-2016.11.18 1 1 179399 534 167.2mb 83.6mb green open .marvel-es-1-2016.11.17 1 1 97604 228 77.9mb 38.9mb green open shakespeare 5 1 111396 0 37.3mb 18.6mb green open estestdbnew 5 1 300000 0 9.3mb 4.6mb green open logstash-2015.05.18 5 1 4631 0 34.6mb 17.3mb green open logstash-2015.05.19 5 1 4624 0 34.9mb 17.4mb
