Grafana----Nginx日志统计展示
一、 效果展示
二、流程
三、 Nginx 配置
nginx.conf，修改日志格式，指定需要收集的字段及字段分割
http{ ........ log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent"'; ........ }
四、 Filebeat 配置
修改filebeat.yml
egrep -v "^#|^ #|^$" filebeat.yml
filebeat.inputs: - type: log enabled: true paths: - /var/log/nginx/xxx.log fields: file_type: nginx-log fields_under_root: true encoding: utf-8 filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template.settings: index.number_of_shards: 1 setup.kibana: output.logstash: hosts: ["xxx.xxx.xxx.xxx:xxxxx"] processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ - add_kubernetes_metadata: ~
五、 Logstash 配置
安装geoip插件
./bin/logstash-plugin install logstash-filter-geoip
修改logstash.conf
input { beats { host => '0.0.0.0' port => xxxxx codec => plain{ charset => "GBK" } } } filter { if [file_type] == "nginx-log"{ grok { match => [ 'message', '%{IPORHOST:remote_addr} - %{NOTSPACE:remote_user} \[%{HTTPDATE:time_local}\] \"%{WORD:method} %{NOTSPACE:request_uri} %{URIPROTO:proto}/%{NUMBER:httpversion}\" %{NUMBER:status} (?:%{NUMBER:size}|-) %{QS:referrer} %{QS:user_agent}' ] } geoip { source => "remote_addr" fields => ["location","country_name","city_name","region_name"] } mutate { remove_field => ["host", "agent"] } } } output { if [file_type] == "nginx-log"{ elasticsearch { hosts => ["xxx.xxx.xxx.xxx:xxxxx"] index => "%{file_type}-%{+YYYY.MM.dd}" user => 'xxxx' password => 'xxxxxx' } } }
六、 Elasticsearch 配置
没有需要特别配置的，保证运行正常就行
七、 Grafana 添加数据源
八、 Json导出（只展示panels）
"panels": [ { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "#8ae9ffcc", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "id": 20, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "日志行数", "type": "stat" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "id": 16, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "remote_user.keyword", "id": "2", "settings": { "order": "desc", "orderBy": "_count", "size": "0" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "metrics": [ { "field": "remote_addr.keyword", "id": "1", "settings": {}, "type": "cardinality" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "独立访问者", "type": "stat" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "custom": { "align": "center", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-blue", "value": null } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "displayName", "value": "响应次数" } ] }, { "matcher": { "id": "byName", "options": "status.keyword" }, "properties": [ { "id": "displayName", "value": "响应代码" } ] } ] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "id": 13, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "访问次数" } ] }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "status.keyword", "id": "2", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_count", "size": "0" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "响应代码", "type": "table" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "I" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "displayName", "value": "访问量占比" } ] } ] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "id": 18, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.4.5", "targets": [ { "alias": "🤖status: 200", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "0", "timeZone": "Asia/Shanghai", "trimEdges": "0" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "metrics": [ { "id": "1", "type": "count" } ], "query": "status: 200 AND user_agent: baidu", "refId": "A", "timeField": "@timestamp" }, { "alias": "status: 301/302", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "status: [301 TO 302] AND user_agent: baidu", "refId": "B", "timeField": "@timestamp" }, { "alias": "status: 4xx", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "status: 4* AND user_agent: baidu", "refId": "C", "timeField": "@timestamp" }, { "alias": "status: 5xx", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "status: 5* AND user_agent: baidu", "refId": "D", "timeField": "@timestamp" }, { "alias": "", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": true, "metrics": [ { "id": "1", "type": "count" } ], "query": "user_agent: baidu", "refId": "E", "timeField": "@timestamp" }, { "alias": "", "bucketAggs": [ { "field": "@timestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": true, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "F", "timeField": "@timestamp" }, { "conditions": [ { "evaluator": { "params": [ 0, 0 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "datasource": { "type": "__expr__", "uid": "__expr__" }, "downsampler": "mean", "expression": "E", "hide": true, "reducer": "sum", "refId": "G", "settings": { "mode": "" }, "type": "reduce", "upsampler": "fillna" }, { "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "F", "hide": true, "reducer": "sum", "refId": "H", "type": "reduce" }, { "datasource": { "type": "__expr__", "uid": "__expr__" }, "expression": "$G / $H", "hide": false, "refId": "I", "type": "math" } ], "title": "百度机器人", "type": "stat" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 12, "w": 16, "x": 0, "y": 14 }, "id": 4, "options": { "basemap": { "config": { "server": "streets" }, "name": "Layer 0", "type": "esri-xyz" }, "controls": { "mouseWheelZoom": false, "showAttribution": true, "showDebug": false, "showScale": false, "showZoom": true }, "layers": [ { "config": { "showLegend": false, "style": { "color": { "fixed": "light-blue" }, "opacity": 0.4, "rotation": { "fixed": 0, "max": 360, "min": -360, "mode": "mod" }, "size": { "fixed": 5, "max": 15, "min": 2 }, "symbol": { "fixed": "img/icons/marker/circle.svg", "mode": "fixed" }, "textConfig": { "fontSize": 12, "offsetX": 0, "offsetY": 0, "textAlign": "center", "textBaseline": "middle" } } }, "location": { "": { "geohash": "geoip.country_name" }, "gazetteer": "public/gazetteer/countries.json", "latitude": "geoip.location.lat", "longitude": "geoip.location.lon", "lookup": "geoip.country_name", "mode": "coords" }, "name": "Layer 1", "tooltip": false, "type": "markers" } ], "view": { "id": "zero", "lat": 0, "lon": 0, "zoom": 2 } }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "settings": { "size": "10000" }, "type": "raw_data" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "IP分布(1万)", "type": "geomap" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "custom": { "align": "center", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-blue", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "displayName", "value": "访问次数" } ] }, { "matcher": { "id": "byName", "options": "geoip.country_name.keyword" }, "properties": [ { "id": "displayName", "value": "国家" } ] }, { "matcher": { "id": "byName", "options": "访问次数" }, "properties": [ { "id": "custom.width", "value": 380 } ] } ] }, "gridPos": { "h": 12, "w": 8, "x": 16, "y": 14 }, "id": 10, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "访问次数" } ] }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "geoip.country_name.keyword", "id": "2", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_count", "size": "10" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "IP国家排名(前十)", "type": "table" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "custom": { "align": "center", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-blue", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "displayName", "value": "访问次数" } ] }, { "matcher": { "id": "byName", "options": "remote_addr.keyword" }, "properties": [ { "id": "displayName", "value": "IP" } ] } ] }, "gridPos": { "h": 12, "w": 10, "x": 0, "y": 26 }, "id": 9, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "访问次数" } ] }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "remote_addr.keyword", "id": "2", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_count", "size": "10" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "IP来源排名(前十)", "type": "table" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "custom": { "align": "center", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-blue", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "displayName", "value": "访问次数" } ] }, { "matcher": { "id": "byName", "options": "referrer.keyword" }, "properties": [ { "id": "displayName", "value": "跳转来源" } ] }, { "matcher": { "id": "byName", "options": "跳转来源" }, "properties": [ { "id": "custom.width", "value": 466 } ] } ] }, "gridPos": { "h": 12, "w": 14, "x": 10, "y": 26 }, "id": 11, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "访问次数" } ] }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "referrer.keyword", "id": "2", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_count", "size": "10" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "referrer排名(前十)", "type": "table" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "custom": { "align": "center", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-blue", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "displayName", "value": "访问次数" } ] }, { "matcher": { "id": "byName", "options": "request_uri.keyword" }, "properties": [ { "id": "displayName", "value": "URI" } ] } ] }, "gridPos": { "h": 12, "w": 10, "x": 0, "y": 38 }, "id": 14, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "访问次数" } ] }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "request_uri.keyword", "id": "2", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_count", "size": "0" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "请求资源排行", "type": "table" }, { "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "custom": { "align": "center", "displayMode": "auto", "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "light-blue", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.displayMode", "value": "gradient-gauge" }, { "id": "displayName", "value": "访问次数" } ] }, { "matcher": { "id": "byName", "options": "user_agent.keyword" }, "properties": [ { "id": "displayName", "value": "UA" } ] } ] }, "gridPos": { "h": 12, "w": 14, "x": 10, "y": 38 }, "id": 12, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "访问次数" } ] }, "pluginVersion": "8.4.5", "targets": [ { "alias": "", "bucketAggs": [ { "field": "user_agent.keyword", "id": "2", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_count", "size": "10" }, "type": "terms" } ], "datasource": { "type": "elasticsearch", "uid": "sGPUvsQ7k" }, "hide": false, "metrics": [ { "id": "1", "type": "count" } ], "query": "", "refId": "A", "timeField": "@timestamp" } ], "title": "UA排名(前十)", "type": "table" } ]
本文来自博客园，作者：ヾ(o◕∀◕)ﾉヾ，转载请注明原文链接：https://www.cnblogs.com/Jupiter-blog/p/16225138.html
【推荐】国内首个AI IDE，深度理解中文开发场景，立即下载体验Trae
【推荐】编程新体验，更懂你的AI，立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包，你的智能百科全书，全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell：AI 加持，快人一步
· DeepSeek 开源周回顾「GitHub 热点速览」
· 物流快递公司核心技术能力-地址解析分单基础技术分享
· .NET 10首个预览版发布：重大改进与新特性概览！
· AI与.NET技术实操系列（二）：开始使用ML.NET
· 单线程的Redis速度为什么快？