filebeat配置

原始日志：

Nov 19 10:10:32 10.0.7.32 date=2024-11-19,time=10: 10:32.980,device_id=FE400FT9200213,log_id=0200024133,type=statistics,pri=information, session_id="4AJ2wt024131-4AJ2AWwv024131",client_name="",client_ip="10.0.1.9",client_cc="ZZ",     dst_ip="10.0.1.9",from="chenxig@yx.com",hfrom="chenxi@yx.com",to="mo20@163.com",polid="3:3:2:SYSTEM",domain="yx.com",mailer="mta",resolved="FAIL",src_type="int",direction="out",virus="",                 disposition="Accept",classifier="Not Spam",message_length="12822",subject="回复: 请",message_id="d0483916fa45d6f3b6a6ea@yx.com",recv_time="",notif_delay="0",scan_time="0.000518",xfer_time="0.002557",          srcfolder="",read_status=""    

 

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /data1/log/fortimail/*.log
  processors:
    - script:
        lang: javascript
        id: remove_double_quotes   #删除原始日志message中的双引号
        source: |
          function process(event) {
            var message = event.Get("message");
            if (message !== undefined) {
              message = message.replace(/"/g, "");
              event.Put("message", message);
            }
          }
 #   - replace:   #该方法没办法将双引号删除，只能将其替换为空格，replacement不能设置为空
 #       fields:
 #         - field: message
 #           pattern: "\""
 #           replacement: ' ' 
 #     ignore_missing: false
 #     fail_on_error: true
    - dissect:
        tokenizer: '%{logmonth} %{logday} %{logtime} %{ip} date=%{logdate},time=%{logtime},device_id=%{device_id},log_id=%{log_id},type=%{type},pri=%{pri}, session_id=%{session_id},client_name=%{client_name},client_ip=%{client_ip},client_cc=%{client_cc},dst_ip=%{dst_ip},from=%{from},hfrom=%{hfrom},to=%{to},polid=%{polid},domain=%{domain},mailer=%{mailer},resolved=%{resolved},src_type=%{src_type},direction=%{direction},virus=%{virus},disposition=%{disposition},classifier=%{classifier},message_length=%{message_length},subject=%{subject},message_id=%{message_id},recv_time=%{recv_time},notif_delay=%{notif_delay},scan_time=%{scan_time},xfer_time=%{xfer_time},srcfolder=%{srcfolder},read_status=%{read_status}'
        field:
        target_prefix: log
    - drop_fields:
        fields: ["message", "log.file" ,"log.device_id" ,"log.log_id" ,"log.type", "log.pri", "log.session_id", "log.client_name", "log.polid", "log.src_type", "log.direction", "log.message_length", "log.message_id", "log.scan_time", "log.xfer_time", "log.srcfolder", "log.mailer" ,"log.offset" ,"log.logmonth" ,"log.logday" ,"log.read_status" ,"log.recv_time"]

# ================================= Processors =================================
processors:
  #- add_host_metadata:
  #    when.not.contains.tags: forwarded
  #- add_cloud_metadata: ~
  #- add_docker_metadata: ~
  #- add_kubernetes_metadata: ~
  #"ecs", "agent", "host", "input"字段在前面的inputs中没办法删除，需要在全局配置中进行删除
  - drop_fields:
     fields: ["ecs", "agent", "host", "input"]

# ================================== Outputs ===================================

# Configure what output to use when sending the data collected by the beat.

output.file:
  path: "/data1/log/fortimail02"
  filename: "fortimail"