#在filebeat服务器安装nginx并且配置json格式的日志
root@ubuntu:~# cat /etc/nginx/nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 768;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
log_format access_json '{"@timestamp":"$time_iso8601",'#这里是为了让nginx的日志时间覆盖掉es里面的timestamp的时间
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"uri":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"tcp_xff":"$proxy_protocol_addr",'
'"http_user_agent":"$http_user_agent",'
'"status":"$status"}';
access_log /var/log/nginx/access_json.log access_json ;
error_log /var/log/nginx/error.log;
gzip on;
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
#重启nginx
systemctl restart nginx.service
#在10.0.0.47安装redis
yum install -y redis
#配置redis
vim /etc/redis/redis.conf
bind 0.0.0.0
requirepass 123456
#启动redis
[root@redis ~]#systemctl restart redis
#记得暂时关闭rsyslog因为日志太多
systemctl stop rsyslog.service syslog.socket
#配置filebeat收集nginx的日志和超级系统的日志
cat >/etc/filebeat/filebeat.yml<<'EOF'
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access_json.log
json.keys_under_root: true #默认False会将json数据存储至message，改为true则会独立message外存储
json.overwrite_keys: true #设为true,覆盖默认的message字段，使用自定义json格式中的key
tags: ["nginx-access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["nginx-error"]
- type: log
enabled: true
paths:
- /var/log/syslog
tags: ["syslog"]
output.redis:
hosts: ["10.0.0.47:6379"]
password: "123456"
db: "0"
key: "filebeat" #所有日志都存放在key名称为filebeat的列表中，llen filebeat可查看长度，即日志记录数
EOF
#logstash是根据tags来区分不同的日志发送到es实现建立不同的索引，所以filebeat要配置tage
#清空nginx的日志
root@ubuntu:~# >/var/log/nginx/access_json.log
root@ubuntu:~# >/var/log/nginx/error.log
#启动filebeat
systemctl restart filebeat.service
#查看redis的key
root@10:/var/log# redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> KEYS *
1) "filebeat"
127.0.0.1:6379> type filebeat
list
127.0.0.1:6379> llen filebeat
(integer) 17047
127.0.0.1:6379> llen filebeat
(integer) 17047
127.0.0.1:6379> llen filebeat
(integer) 17047
#取出日志信息查看
127.0.0.1:6379> lpop filebeat
"{\"@timestamp\":\"2023-08-10T14:05:34.000Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.17.5\"},\"size\":612,\"status\":\"200\",\"responsetime\":0,\"domain\":\"10.0.0.74\",\"xff\":\"-\",\"uri\":\"/index.nginx-debian.html\",\"clientip\":\"10.0.0.74\",\"tags\":[\"nginx-access\"],\"input\":{\"type\":\"log\"},\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_user_agent\":\"curl/7.68.0\",\"referer\":\"-\",\"agent\":{\"version\":\"7.17.5\",\"hostname\":\"ubuntu\",\"ephemeral_id\":\"b956afe8-085f-4b7f-914e-533880abf106\",\"id\":\"12aede24-e6de-40b9-a884-583d5996bd74\",\"name\":\"ubuntu\",\"type\":\"filebeat\"},\"tcp_xff\":\"-\",\"log\":{\"offset\":39904,\"file\":{\"path\":\"/var/log/nginx/access_json.log\"}},\"ecs\":{\"version\":\"1.12.0\"},\"http_host\":\"10.0.0.74\",\"host\":{\"name\":\"ubuntu\"}}"
127.0.0.1:6379> lpop filebeat
"{\"@timestamp\":\"2023-08-10T14:05:34.000Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.17.5\"},\"ecs\":{\"version\":\"1.12.0\"},\"uri\":\"/index.nginx-debian.html\",\"size\":612,\"referer\":\"-\",\"status\":\"200\",\"domain\":\"10.0.0.74\",\"responsetime\":0,\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"ubuntu\"},\"xff\":\"-\",\"clientip\":\"10.0.0.74\",\"upstreamhost\":\"-\",\"tcp_xff\":\"-\",\"upstreamtime\":\"-\",\"http_host\":\"10.0.0.74\",\"tags\":[\"nginx-access\"],\"agent\":{\"hostname\":\"ubuntu\",\"ephemeral_id\":\"b956afe8-085f-4b7f-914e-533880abf106\",\"id\":\"12aede24-e6de-40b9-a884-583d5996bd74\",\"name\":\"ubuntu\",\"type\":\"filebeat\",\"version\":\"7.17.5\"},\"http_user_agent\":\"curl/7.68.0\",\"log\":{\"offset\":40222,\"file\":{\"path\":\"/var/log/nginx/access_json.log\"}}}"
#触发nginx的访问日志在检查redis的filebeat的key的数量
root@10:~# curl 10.0.0.74 -I
HTTP/1.1 200 OK
Server: nginx/1.18.0 (Ubuntu)
Date: Sat, 12 Aug 2023 18:25:04 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Wed, 09 Aug 2023 15:29:54 GMT
Connection: keep-alive
ETag: "64d3b0f2-264"
Accept-Ranges: bytes
root@10:~# curl 10.0.0.74/XX -I
HTTP/1.1 404 Not Found
Server: nginx/1.18.0 (Ubuntu)
Date: Sat, 12 Aug 2023 18:25:06 GMT
Content-Type: text/html
Content-Length: 162
Connection: keep-alive
#检查redis
127.0.0.1:6379> llen filebeat
(integer) 17045
127.0.0.1:6379> llen filebeat
(integer) 17045
127.0.0.1:6379> llen filebeat
(integer) 17047
127.0.0.1:6379> llen filebeat
(integer) 17047
127.0.0.1:6379> llen filebeat
(integer) 17047
#配置logstash收集日志输出日志到es
cat >/etc/logstash/conf.d/redis_geoip_to_es.conf.conf<<'EOF'
input {
redis {
host => '10.0.0.47'
port => "6379"
password => "123456"
db => "0" #在0号数据库抓取日志
data_type => 'list' #数据类型为list
key => "filebeat" #把收集到的日志传输给filebeat的key
}
}
filter {
if "nginx-access" in [tags] {
# geoip {
# source => "clientip"
# target => "geo"
# }
}
}
output {
if "syslog" in [tags] {
elasticsearch {
hosts => ["10.0.0.70:9200","10.0.0.71:9200","10.0.0.72:9200"]
index => "syslog-%{+YYYY.MM.dd}"
}
}
if "nginx-access" in [tags] {
elasticsearch {
hosts => ["10.0.0.70:9200","10.0.0.71:9200","10.0.0.72:9200"]
index => "logstash-nginx-accesslog-%{+YYYY.MM.dd}"
template_overwrite => true
}
}
if "nginx-error" in [tags] {
elasticsearch {
hosts => ["10.0.0.70:9200","10.0.0.71:9200","10.0.0.72:9200"]
index => "logstash-nginx-errorlog-%{+YYYY.MM.dd}"
template_overwrite => true
}
}
}
EOF
#做语法检查
logstash -f /etc/logstash/conf.d/redis_geoip_to_es.conf.conf -t
#启动
logstash -f /etc/logstash/conf.d/redis_geoip_to_es.conf.conf -r
#检查redis的key
root@10:/var/log# redis-cli -a 123456
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
127.0.0.1:6379> llen filebeat
(integer) 0
127.0.0.1:6379> llen filebeat
(integer) 0
#已经被logstash提取完毕
#在网页检查es集群
