nginx + graylog 对于日志进行管理的一个实践
以下整理一个自己结合ngin+graylog 进行日志处理的实践，可以参考
日志参考玩法
参考配置
- log format
参考如下，可以配置一些符合自己业务的log format 不同业务配置使用
log_format main '$remote_addr - $remote_user [$time_local] requesthost:"$http_host"; "$request" requesttime:"$request_time"; '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
log_format graylog2_json escape=json '{ "timestamp": "$time_local", '
'"remote_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"request_body":"$request_body",'
'"source_ip": "$http_x_forwarded_for",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"upstream_response_time": "$upstream_response_time",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"http_referrer": "$http_referer", '
'"http_user_agent": "$http_user_agent",'
'"realip":"$realip_remote_addr"}';
log_format graylog3_json escape=json '{ "timestamp": "$time_local", '
'"remote_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"request_body":"$request_body",'
'"response_body":"$resp_body",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"source_ip": "$http_x_forwarded_for",'
'"upstream_response_time": "$upstream_response_time",'
'"http_referrer": "$http_referer", '
'"http_user_agent": "$http_user_agent",'
'"realip":"$realip_remote_addr"}';
log_format graylog4_json escape=json '{ "timestamp": "$time_local", '
'"remote_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"request_body":"$request_body",'
'"response_body":"$resp_body",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"source_ip": "$http_x_forwarded_for",'
'"source_ip_fromf5": "$http_myip",'
'"http_referrer": "$http_referer", '
'"upstream_response_time": "$upstream_response_time",'
'"http_user_agent": "$http_user_agent",'
'"realip":"$realip_remote_addr"}';
log_format graylog5_json escape=json '{ "timestamp": "$time_local", '
'"remote_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"source_ip": "$http_x_forwarded_for",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"upstream_response_time": "$upstream_response_time",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"http_referrer": "$http_referer", '
'"http_user_agent": "$http_user_agent",'
'"realip":"$realip_remote_addr"}';
log_format graylog6_json escape=json '{ "timestamp": "$time_local", '
'"remote_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"request_body":"$request_body",'
'"response_body":"$resp_body",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"source_ip": "$http_x_forwarded_for",'
'"upstream_response_time": "$upstream_response_time",'
'"http_referrer": "$http_referer", '
'"http_cookie": "$http_cookie",'
'"http_user_agent": "$http_user_agent",'
'"realip":"$realip_remote_addr"}';
log_format log2 escape=json '$remote_addr $time_local $request_method $request_uri $status $request_time "$request_body"';
- 公共部分
user root;
worker_processes auto;
worker_cpu_affinity auto;
error_log logs/error.log error;
error_log syslog:server=<ssylog serbver>:12407,tag=lb_ingress_error error;
events {
use epoll;
worker_connections 655360;
}
http {
include common/*.conf;
include app/*.conf;
}
- 业务系统
upstream xxxxxx {
# simple round-robin
least_conn;
server xxxxx:80;
#check interval=1000 rise=2 fall=5 timeout=1000 type=http;
#check_http_send "HEAD / HTTP/1.0\r\n\r\n";
#check_http_expect_alive http_2xx http_3xx;
}
server {
listen 80;
server_name xxxxx;
# 按需配置 access_log
access_log syslog:server=xxxxx:12401 graylog3_json;
location / {
return 301 https://$host$request_uri;
}
}
server {
listen 443 ssl http2;
server_name xxxxxxx;
ssl_certificate ssl/xxxxx.pem;
ssl_certificate_key ssl/xxxxxx.key;
# 按需配置 access_log
access_log syslog:server=xxxxxx:12401 graylog3_json;
location / {
# 按需配置 access_log
access_log syslog:server=xxxxxx:12401 graylog3_json;
// 基于openresty 进行response 数据处理，按需配置
body_filter_by_lua_block {
local resp_body = string.sub(ngx.arg[1], 1, 1000)
ngx.ctx.buffered = string.sub((ngx.ctx.buffered or "") .. resp_body, 1, 1000)
-- arg[2] is true if this is the last chunk
if ngx.arg[2] then
ngx.var.resp_body = ngx.ctx.buffered
end
}
proxy_set_header Host $http_hotst;
proxy_set_header X-Forwarded-For $remote_addr;
client_body_buffer_size 10M;
client_max_body_size 10G;
proxy_buffers 1024 4k;
proxy_read_timeout 300;
proxy_pass http://xxxxxx;
}
}
报警处理
graylog 支持alert（4.0 之后比较方便）
- 参考图
- 简单说明
基于graylog 的stream 以及rule 将不同的业务系统日志分散到不同的es 存储中，对于alert 会基stream 以及查询规则进行消息的通知，通知模式包含了email webhook
说明
基于graylog 比较完整的日志处理模式，对于nginx 以及一些业务系统的日志监控还是比较方便的，graylog 包含了比较完整的权限体系以及灵活的数据存储处理，是一个很不错的日志存储，检索以及报警处理平台，以上是自己的一个实践，上边只是一个简单的说明，实际上我以前也大概写过一些，可以参考
参考资料
https://go2docs.graylog.org/5-0/what_is_graylog/what_is_graylog.htm
https://www.cnblogs.com/rongfengliang/p/11251458.html
