Filebeat - 安装部署
Elastic官网： https://www.elastic.co/cn/elastic-stack/
下载安装包
搜索下载: https://www.elastic.co/cn/downloads/past-releases#elasticsearch
注意： 整套组件需要版本一致，当前案例版本为 6.7.0
**elasticsearch**: wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.7.0.tar.gz
**kibana**: wget https://artifacts.elastic.co/downloads/kibana/kibana-6.7.0-linux-x86_64.tar.gz
**logstash**: wget https://artifacts.elastic.co/downloads/logstash/logstash-6.7.0.tar.gz
**filebeat**: wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.7.0-linux-x86_64.tar.gz
Filebeat安装部署
1.上传压缩包 filebeat-6.7.0-linux-x86_64.tar.gz
2.解压 tar -zxvf filebeat-6.7.0-linux-x86_64.tar.gz
3.配置 vim filebeat.yml
4.启动
A : ./filebeat -e -c filebeat.yml
-c：配置文件位置
-path.logs：日志位置
-path.data：数据位置
-path.home：家位置
-e：关闭日志输出
-d 选择器：启用对指定选择器的调试。 对于选择器，可以指定逗号分隔的组件列表，也可以使用-d“*”为所有组件启用调试.例如，-d“publish”显示所有“publish”相关的消息。
./filebeat -e -c filebeat.yml -d “publish”
B : 后台启动filebeat
将所有标准输出及标准错误输出到/dev/null空设备，即没有任何输出
nohup ./filebeat -e -c filebeat.yml >/dev/null 2>&1 &
输入日志到filebeat.log 文件中
nohup ./filebeat -e -c filebeat.yml > filebeat.log &
C.shell 启动
#!/bin/bash
nohup bin/logstash -f ./config/logstash-syne.conf --config.reload.automatic >./logs/out.log &
5.停止filebeat：ps -ef |grep filebeat, kill -9 pid
6.使用
Filebeat 过滤 使用:
如果想对采集的内容进行预处理(过滤等),比如从日志中提取某些字段filebeat不像logstash那么灵活需要借助es的pipeline，而此处主要是将@timestamps时间修改日志的时间（默认是采集的时间）
（1）在es中创建一个pipeline，timestamp-pipeline-id 是唯一的
PUT _ingest/pipeline/timestamp-pipeline-id
{
"description": "timestamp pipeline",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{TIMESTAMP_ISO8601:timestamp} "
]
}
},
{
"date": {
"field": "timestamp",
"formats": [
"yyyy-MM-dd HH:mm:ss.SSS"
]
},
"remove": {
"field": "timestamp"
}
}
],
"on_failure": [
{
"set": {
"field": "_index",
"value": "failed-{{ _index }}"
}
}
]
}
这样就完成了所有的工作。这时启动filebeat, 如果如出以下错误信息
ERROR pipeline/output.go:92 Failed to publish events: temporary bulk send failure
大概率是因为你发送的日志格式无法与grok表达式匹配，修改processor定义json即可。也可以在启动filebeat时添加-d "*"参数来查看具体的错误原因。
7.案例
## 模式二: filebeat -> ES
//ES 添加管道语句
PUT _ingest/pipeline/timestamp-pipeline-id
{
"description" : "timestamp pipeline",
"processors": [
{
"grok": {
"field": "message",
"patterns": ["\\[%{HOSTNAME:log_service_name}:%{HOSTPORT:service_ip}\\]\\s*%{TIMESTAMP_ISO8601:log_time}\\s*%{LOGLEVEL:log_level}\\s*%{INT:pid}\\s*\\[%{GREEDYDATA:request_trace_id}\\]\\s*\\[%{GREEDYDATA:thread_id}\\]\\s*%{NOTSPACE:java_class}\\s*Http Request: CommonLog{createBy=%{GREEDYDATA:create_by},\\s*updateBy=%{GREEDYDATA:update_by},\\s*createTime=%{GREEDYDATA:create_time},\\s*updateTime=%{GREEDYDATA:update_time},\\s*isDeleted='%{GREEDYDATA:is_deleted}',\\s*id=%{GREEDYDATA:data_id},\\s*type='%{GREEDYDATA:type}',\\s*traceId='%{GREEDYDATA:trace_id}',\\s*serviceName='%{GREEDYDATA:service_name}',\\s*title='%{GREEDYDATA:title}',\\s*operation='%{GREEDYDATA:operation}',\\s*method='%{GREEDYDATA:method}',\\s*url='%{GREEDYDATA:url}',\\s*params='%{GREEDYDATA:params}',\\s*ip='%{GREEDYDATA:ip}',\\s*executeTime=%{GREEDYDATA:execute_time},\\s*location='%{GREEDYDATA:location}',\\s*tenantId=%{GREEDYDATA:tenant_id},\\s*exception='%{GREEDYDATA:exception}',\\s*createName='%{GREEDYDATA:create_name}',\\s*updateName='%{GREEDYDATA:update_name}',\\s*operationType='%{GREEDYDATA:operation_type}',\\s*customsId='%{GREEDYDATA:customs_id}',\\s*parkId=%{GREEDYDATA:park_id},\\s*companyId=%{GREEDYDATA:company_id}}"]
}
},
{
"date": {
"field": "log_time",
"formats": ["yyyy-MM-dd HH:mm:ss.SSS"],
"timezone": "Asia/Shanghai",
"target_field": "@timestamp"
}
}
]
}
//测试管道语句
POST _ingest/pipeline/_simulate
{
"pipeline": {
"description" : "timestamp pipeline",
"processors": [
{
"grok": {
"field": "message",
"patterns": ["\\[%{HOSTNAME:log_service_name}:%{HOSTPORT:service_ip}\\]\\s*%{TIMESTAMP_ISO8601:log_time}\\s*%{LOGLEVEL:log_level}\\s*%{INT:pid}\\s*\\[%{GREEDYDATA:request_trace_id}\\]\\s*\\[%{GREEDYDATA:thread_id}\\]\\s*%{NOTSPACE:java_class}\\s*Http Request: CommonLog{createBy=%{GREEDYDATA:create_by},\\s*updateBy=%{GREEDYDATA:update_by},\\s*createTime=%{GREEDYDATA:create_time},\\s*updateTime=%{GREEDYDATA:update_time},\\s*isDeleted='%{GREEDYDATA:is_deleted}',\\s*id=%{GREEDYDATA:data_id},\\s*type='%{GREEDYDATA:type}',\\s*traceId='%{GREEDYDATA:trace_id}',\\s*serviceName='%{GREEDYDATA:service_name}',\\s*title='%{GREEDYDATA:title}',\\s*operation='%{GREEDYDATA:operation}',\\s*method='%{GREEDYDATA:method}',\\s*url='%{GREEDYDATA:url}',\\s*params='%{GREEDYDATA:params}',\\s*ip='%{GREEDYDATA:ip}',\\s*executeTime=%{GREEDYDATA:execute_time},\\s*location='%{GREEDYDATA:location}',\\s*tenantId=%{GREEDYDATA:tenant_id},\\s*exception='%{GREEDYDATA:exception}',\\s*createName='%{GREEDYDATA:create_name}',\\s*updateName='%{GREEDYDATA:update_name}',\\s*operationType='%{GREEDYDATA:operation_type}',\\s*customsId='%{GREEDYDATA:customs_id}',\\s*parkId=%{GREEDYDATA:park_id},\\s*companyId=%{GREEDYDATA:company_id}}"]
}
},
{
"date": {
"field": "log_time",
"formats": ["yyyy-MM-dd HH:mm:ss.SSS"],
"timezone": "Asia/Shanghai",
"target_field": "@timestamp"
}
}
]
},
"docs": [
{
"_index": "syne_sys_log",
"_id": "id",
"_source": {
"message":"[basic:192.168.0.107:21001] 2021-03-26 11:47:01.344 INFO 19432 [] [http-nio-21001-exec-1] com.kzkj.core.log.aspect.LogAspect Http Request: CommonLog{createBy=22, updateBy=null, createTime=2021-03-26T11:47:01.344, updateTime=null, isDeleted='null', id=1382619049657229313, type='1', traceId='6c82c964-8d69-48a6-a3ba-e06085d25aa7', serviceName='basic', title='获取货主管理列表', operation='ResultPage(total=1, pages=1, size=10, current=1)', method='POST', url='/owner/list', params='[{\"current\":1,\"hitCount\":false,\"optimizeCountSql\":true,\"order\":\"DESC\",\"orders\":[],\"pages\":0,\"records\":[],\"searchCount\":true,\"size\":10,\"total\":0}]', ip='127.0.0.1', executeTime=68, location='null', tenantId=null, exception='null', createName='pp1', updateName='null', operationType='1', customsId='0414',parkId=1370275456463519723,companyId=1}"
}
}
]
}
//业务日志表结构
PUT syne_sys_log
{
"settings": {
"number_of_shards": 1,
"number_of_replicas":1,
"index.refresh_interval": "1s"
},
"mappings": {
"doc": {
"properties": {
"id": { "type": "keyword" },
"service_ip": { "type": "keyword" },
"service_name": { "type": "keyword"},
"log_level": { "type": "keyword"},
"source": { "type": "keyword"},
"message": { "type": "keyword"},
"log_time": { "type": "keyword"},
"content": { "type": "keyword"},
"thread_id": { "type": "keyword"},
"pid": { "type": "keyword"},
"request_trace_id": { "type": "keyword"},
"java_class": { "type": "keyword"},
"data_id": { "type": "keyword"},
"tenant_id": { "type": "keyword"},
"exception": { "type": "keyword"},
"operation_type": { "type": "keyword"},
"log_service_name": { "type": "keyword"},
"type": { "type": "keyword"},
"title": { "type": "keyword"},
"update_name": { "type": "keyword"},
"create_by": { "type": "keyword"},
"update_time": { "type": "keyword"},
"is_deleted": { "type": "keyword"},
"execute_time": { "type": "keyword"},
"update_by": { "type": "keyword"},
"trace_id": { "type": "keyword"},
"create_time": { "type": "keyword"},
"method": { "type": "keyword"},
"ip": { "type": "keyword"},
"params": { "type": "keyword"},
"url": { "type": "keyword"},
"location": { "type": "keyword"},
"operation": { "type": "keyword"},
"create_name": { "type": "keyword"}
}
}
}
}
