filebeat 配置采集nginx 日志

filebeat 配置nginx 日志采集

• filebeat 采集需求

1.需要将以往30 天的日志输出到es，并且以时间按天展示
2.将不同的时间字段解析出来，输出到es

• nginx 配置json 日志

  log_format log_json '{ "remoteAddr": "$clientRealIp", '
'"date_timeLocal": "$time_local", '
'"remoteUser": "$remote_user", '
'"requestType": "$request_method", '
'"requestUrl": "$uri", '
'"URIPROTO": "$server_protocol", '
'"args": "$args", '
'"scheme": "$scheme", '
'"long_status": $status, '
'"long_bodyBytesSent": $body_bytes_sent, '
'"httpReferer": "$http_referer", '
'"httpUserAgent": "$http_user_agent", '
'"upstream_addr": "$upstream_addr", '
'"request_time": "$request_time",'
'"http_website": "$http_website",'
'"http_g_id": "$http_g_id",'
'"http_s_id": "$http_s_id",'
'"http_u_id": "$http_u_id"'
' }';

• nginx server 配置日志格式

access_log /export/home/logs/production/access.log log_json;
error_log /export/home/logs/production/error.log warn;

• nginx 日志样式

{ "remoteAddr": "12.11.11.111", "date_timeLocal": "24/Aug/2023:00:00:00 +0800", "remoteUser": "-", "requestType": "POST", "requestUrl": "/api/v1/words/pc/semantic/defi/", "URIPROTO": "HTTP/1.1", "args": "-", "scheme": "http", "long_status": 200, "long_bodyBytesSent": 41, "httpReferer": "https://xxx/wantWordsResult?lang=zh&query=%E5%A4%B9%E5%B8%A6%E7%A7%81%E8%B4%A7&category=1001", "httpUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.54", "upstream_addr": "192.168.26.178:8087", "request_time": "0.232","http_website": "-","http_g_id": "24e023cf-ab6a-4894-b30b-83cc749d778d","http_s_id": "YRTyTyB7-4687-4336-4f3s-yB167U92KY80","http_u_id": "64c06effd35d7c4b9c99e924" }
{ "remoteAddr": "12.11.11.111", "date_timeLocal": "24/Aug/2023:00:00:01 +0800", "remoteUser": "-", "requestType": "GET", "requestUrl": "/api/v1/words/pc/history/", "URIPROTO": "HTTP/1.1", "args": "lang=zh", "scheme": "http", "long_status": 200, "long_bodyBytesSent": 41, "httpReferer": "https://xxx/", "httpUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15", "upstream_addr": "192.168.26.178:8087", "request_time": "0.016","http_website": "-","http_g_id": "7c767a7b-7a77-482d-b2f6-3aa7951ea5b9","http_s_id": "rKY4Y4Pw-9204-4639-50CA-4P16sO92Pr80","http_u_id": "-" }

• 配置filebeat 日志采集

[root@dev-test-lingowhale filebeat]# cat filebeat.yml
# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: log
  id: shenyandayinginx-id
  enabled: true
  paths:
    - /lingowhale/k8snode*/project/volume-frontend/prod-frontend/access*.log
  fields:
    product: shenyandayi_nginx
  json.keys_under_root: true
  json.overwrite_keys: true
# ============================== Filebeat modules ==============================
filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true

# ======================= Elasticsearch template setting =======================
setup.template.enabled: false
setup.ilm.enabled: false

# =================================== Kibana ===================================

# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
# This requires a Kibana endpoint configuration.
setup.kibana:
  host: "10.0.0.2:5601"

# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
  # Array of hosts to connect to.
  hosts: ["10.0.0.0:9200"]
  indices:
    - index: "prod-nginx"
# ================================= Processors =================================
#注意： 匹配日志格式，有两种日志格式需要解析，第一种是 2023-07-19T00:00:02+08:00 、第二种是 24/Aug/2023:00:00:01 +0800 
#timestamp 处理器，将nginx 30天的日志 按照 date_timeLocal 字段进行解析，输出到es 里会展示真实日志里面当时的时间，而不是现在的采集时间

processors:
  - timestamp:
      field: date_timeLocal
      timezone: Asia/Shanghai
      layouts:
        - '2006-01-02T15:04:05Z'
        - '2006-01-02T15:04:05.999Z'
        - '2006-01-02T15:04:05.999-07:00'
        - '02/Jan/2006:15:04:05 +0800'
      test:
        - '2019-06-22T16:33:51Z'
        - '2019-11-18T04:59:51.123Z'
        - '02/Jan/2006:15:04:05 +0800'
        - '2020-08-03T07:10:20.123456+02:00'
  - drop_fields:
      fields: ["agent","offset", "prospector", "source", "input", "beat","date_timeLocal"]

• filebeat 启动并设置定时任务

[root@dev-test-xxxmanagelog]# cat /opt/scripts/monitorlog.sh
#!/bin/sh
process_num=`ps -ef |grep filebeat.yml |grep -v 'grep' |wc -l`
if [ ${process_num} -eq 0 ];then
  cd /export/filebeat && nohup ./filebeat -e -c filebeat.yml >> /export/filebeat/filebeat.log 2>&1 &
else
  echo "进程运行---"
fi


# 配置定时任务
[root@dev-test-lingowhale managelog]# crontab  -l
* * * * * /bin/bash /opt/scripts/monitorlog.sh > /dev/null 2>&1

• kibana 日志展示
  

• kibana 开发工具使用

# 删除索引中某一字段的数据
POST /prod-nginx-*/_delete_by_query
{
  "query": {
    "match": {
      "fields.product": "deeplang_test"
    }
  }
}