Graylog+elasticsearch+mongodb集群+nginx负载均衡前端
网上有张图画的很好，搜索有关它的配置文章，google里有几篇英文的，都是依靠haproxy等或别的什么实现，没有纯粹的Graylog+elasticsearch+mongodb集群，项目需要，只有自己摸索一下了，搞好发现其实也很简单。
整个结构大概就是上图样子，nginx的负载均衡可以参考我前面的文章。将后端的web集群换成graylog集群地址就可以了。
下面是4台集群的配置效果图，完成后的效果是随便访问那个IP，都可以访问到所有的日志，节点可以根据需求任意添加，无论graylog，es,还是mongodb都可以。
配置过程如下，难点在于graylog的集群，安装软件的过程可以参考前面的安装过程。
========================================elasticsearch=================================
#vim /etc/elasticsearch/elasticsearch.yml
node.master: true #两台机器设置成master，其余节点此项不写
network.bind_host: 192.168.1.231 #绑定实际的服务器IP地址，不是集群的IP
cluster.name: graylog #与/etc/graylog/server/server.conf 中的elasticsearch_cluster_name 集群名称必须相同
discovery.zen.minimum_master_nodes: 2 #4台机器就2个够了
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ["192.168.1.231:9300","192.168.1.232:9300","192.168.1.233:9300","192.168.1.234:9300"]
=================================================mongodb========================================
vim /etc/mongod.conf #所有节点配置文件相同
replication:
replSetName: graylog
systemLog:
destination: file
logAppend: true
path: /var/log/mongodb/mongod.log
storage:
dbPath: /var/lib/mongo
journal:
enabled: true
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod.pid # location of pidfile
net:
port: 27017
#mongo//进入数据库配置
rs.initiate()
rs.add("192.168.1.231:27017");
rs.add("192.168.1.232:27017");
rs.addArb("192.168.1.233:27017");
rs.add("192.168.1.234:27017");
graylog:PRIMARY> rs.config(); rs.config();
{
"_id" : "graylog",
"version" : 4,
"protocolVersion" : NumberLong(1),
"members" : [
{
"_id" : 0,
"host" : "node231:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "192.168.1.232:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 2,
"host" : "192.168.1.234:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
},
{
"_id" : 3,
"host" : "192.168.1.233:27017",
"arbiterOnly" : true,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("578c49834bd6994271fcfad7")
}
}
============================graylog===================================
#vim /etc/graylog/server/server.conf
password_secret =0b4e7a0e5fe84ad35fb5f95b9ceeac790b4e7a0e5fe84ad35fb5f95b9ceeac79
root_password_sha2 =ed02457b5c41d964dbd2f2a609d63fe1bb7528dbe55e1abf5b52c249cd735797
elasticsearch_cluster_name = graylog
elasticsearch_discovery_zen_ping_multicast_enabled = false
elasticsearch_discovery_zen_ping_unicast_hosts = 192.168.1.231:9300,192.168.1.232:9300,192.168.1.233:9300,192.168.1.234:9300
mongodb_uri = mongodb://192.168.1.231:27017,192.168.1.232:27017,192.168.1.233:27017,192.168.1.234:27017/graylog
elasticsearch_shards = 4
elasticsearch_replicas = 1
mongodb_useauth = false #默认为false,可不必添加，生产环境必须
=====================测试=============================
使用curl命令操作elasticsearch
# curl -X GET 'http://192.168.1.231:9200/_cat/nodes'
192.168.1.234 192.168.1.234 7 13 0.06 d * Marrina
192.168.1.233 192.168.1.233 7 21 0.12 d m Banshee
192.168.1.231 192.168.1.231 8 23 0.43 d m Katie Power
192.168.1.232 192.168.1.232 7 22 0.15 d m Dark Phoenix
192.168.1.231 192.168.1.231 20 23 0.43 c - graylog-8562414f-0f9b-48d9-b0c6-9faab367832e
]# curl -X GET 'http://192.168.1.231:9200/_cluster/health?pretty=true'
{
"cluster_name" : "graylog",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 4,
"number_of_data_nodes" : 3,
"active_primary_shards" : 4,
"active_shards" : 4,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
mongodb状态查询
graylog:PRIMARY> rs.status()
{
"set" : "graylog",
"date" : ISODate("2016-07-18T09:14:33.419Z"),
"myState" : 1,
"term" : NumberLong(2),
"heartbeatIntervalMillis" : NumberLong(2000),
"members" : [
{
"_id" : 0,
"name" : "node231:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 19532,
"optime" : {
"ts" : Timestamp(1468833273, 2),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2016-07-18T09:14:33Z"),
"electionTime" : Timestamp(1468813741, 1),
"electionDate" : ISODate("2016-07-18T03:49:01Z"),
"configVersion" : 5,
"self" : true
},
{
"_id" : 1,
"name" : "192.168.1.232:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 8272,
"optime" : {
"ts" : Timestamp(1468833271, 4),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2016-07-18T09:14:31Z"),
"lastHeartbeat" : ISODate("2016-07-18T09:14:31.802Z"),
"lastHeartbeatRecv" : ISODate("2016-07-18T09:14:31.765Z"),
"pingMs" : NumberLong(0),
"syncingTo" : "node231:27017",
"configVersion" : 5
},
{
"_id" : 2,
"name" : "192.168.1.234:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 8257,
"optime" : {
"ts" : Timestamp(1468833271, 4),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2016-07-18T09:14:31Z"),
"lastHeartbeat" : ISODate("2016-07-18T09:14:31.736Z"),
"lastHeartbeatRecv" : ISODate("2016-07-18T09:14:31.802Z"),
"pingMs" : NumberLong(0),
"syncingTo" : "192.168.1.232:27017",
"configVersion" : 5
},
{
"_id" : 3,
"name" : "192.168.1.233:27017",
"health" : 1,
"state" : 7,
"stateStr" : "ARBITER",
"uptime" : 8092,
"lastHeartbeat" : ISODate("2016-07-18T09:14:31.802Z"),
"lastHeartbeatRecv" : ISODate("2016-07-18T09:14:32.548Z"),
"pingMs" : NumberLong(0),
"configVersion" : 5
}
],
"ok" : 1
}
graylog:PRIMARY>
