采集日志工具-fluentbit输出到kafka

fluentbit

官网

安装

[root@logstash system]# cat /etc/yum.repos.d/td-agent-bit.repo
[td-agent-bit]
name = TD Agent Bit
baseurl = https://packages.fluentbit.io/centos/7/$basearch/
gpgcheck=1
gpgkey=https://packages.fluentbit.io/fluentbit.key
enabled=1
yum install td-agent-bit -y

配置

[root@logstash alikafka]# cat main.conf
[SERVICE]
    flush        5
    daemon       Off
    log_level    info
    #parsers_file parsers.conf
    #plugins_file plugins.conf
    #storage.metrics on

@INCLUDE inputs.conf
@INCLUDE outputs.conf

#[FILTER]
#    Name modify
#    Match *
#    Rename log message

[FILTER]
    Name modify
    Match adclick*
    Add Topic_Key adclick

[FILTER]
    Name modify
    Match adshow*
    Add Topic_Key adshow

[root@logstash alikafka]# cat inputs.conf
[INPUT]
    name tail
    Path /data/logs/ads*/ads/ads-*/adClickLog/adclick.log
    Key  message
    DB   /tmp/adclick.db
    Tag  adclick_kafka
    #Read_from_Head true
    Refresh_Interval 10
    #Ignore_Older   10m
    Buffer_Chunk_Size 2048k
    Buffer_Max_Size   2048k

[INPUT]
    name tail
    Path /data/logs/ads*/ads/ads-*/adShowLog/adshow.log
    Key  message
    DB   /tmp/adshow.db
    Tag  adshow_kafka
    Refresh_Interval 10
    Buffer_Chunk_Size 2048k
    Buffer_Max_Size   2048k

[root@logstash alikafka]# cat outputs.conf
[OUTPUT]
    Name        kafka
    Match       *
    Brokers     kafka-01:9092, kafka-02:9092, kafka-03:9092
    Topics      adclick,adshow
    topic_key   Topic_Key
    rdkafka.request.required.acks 1
    rdkafka.linger.ms 0
    rdkafka.batch.size 65536
    rdkafka.message.max.bytes 2097152
    rdkafka.log.connection.close false
    rdkafka.compression.type lz4
    rdkafka.compression.level 6

启动

mv /usr/lib/systemd/system/td-agent-bit.service /usr/lib/systemd/system/fluentbit.service
sed -i 's#td-agent-bit.conf#main#g' /usr/lib/systemd/system/fluentbit.service
systemctl daemon-reload
systemctl start fluentbit.service
#手动调试启动
#/opt/td-agent-bit/bin/td-agent-bit -c /etc/td-agent-bit/main.conf