【原创】fluent-bit安装使用

部署
wget https://github.com/fluent/fluent-bit/archive/refs/tags/v1.7.1.tar.gz
yum install cmake3 gcc gcc-c++ flex bison -y
tar xf v1.7.1.tar.gz
cd fluent-bit-1.7.1/build/
cmake3 ../
make && make install

配置
mkdir -p /usr/local/fluent/conf
vim /usr/local/fluent/conf/fluent-all.conf
[SERVICE]
FLush 5
Daemon off
Log_Level debug
# storage.path /var/log/flb-storage/
# storage.sync normal
# storage.checksum off
# storage.backlog.mem_limit 500M

[INPUT]
# Name dummy
# Dummy {"tool": "fluent", "sub": {"s1": {"s2": "bit"}}}
# Tag testlog
Name forward
Listen 0.0.0.0
Port 24224
#Port 24226
#Tag testlog
#Buffer_Chunk_Size 1M
#Buffer_Max_Size 6M

[OUTPUT]
Name es
Match platform
Host 127.0.0.1
Port 9200
Index platform
Type logs
Include_Tag_key On
Tag_key tag

[OUTPUT]
Name es
Match pull
Host 127.0.0.1
Port 9200
Index pull
Type logs

cat > /usr/local/fluent/start-fluent.sh << EOF
#!/bin/bash
while true;do
if [[ ! `ss -tnl| grep 24224` ]];then
/usr/local/fluent/bin/fluent-bit --config=/usr/local/fluent/conf/fluent-all.conf
fi
done
EOF

 

flunt-mysql.conf  mysql slow.log日志

[SERVICE]
Flush 5
Daemon off
Log_Level debug
Parsers_File parsers.conf

#[PARSER]
# Name mysql_log
# Format regex
# Regex /(?m)^#\s+Time\s?.*\s+#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IPV4:clientip})?\]\s+Id:\s+%{NUMBER:row_id:int}\n#\s+Query_time:\s+%{NUMBER:query_time:float}\s+Lock_time:\s+%{NUMBER:lock_time:float}\s+Rows_sent:\s+%{NUMBER:rows_sent:int}\s+Rows_examined:\s+%{NUMBER:rows_examined:int}\n/

#[PARSER]
# Name mysql_attribues
# Format regex
# Regex /?SET\s+timestamp=%{NUMBER:timestamp};\n\s*(?<sql>(?<action>\w+)([\w.*\W.*])*;)\s*$/

[INPUT]
Name tail
tag mysql-dashuju1
path /root/1.log
Multiline On
Parser_Firstline mysql_log
Parser_1 mysql_attribues

[OUTPUT]
Name stdout
Match mysql-dashuju1

 

vim parsers.conf

[PARSER]
Name mysql_log
Format regex
Regex /(?m)^#\s+Time\s?.*\s+#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IPV4:clientip})?\]\s+Id:\s+%{NUMBER:row_id:int}\n#\s+Query_time:\s+%{NUMBER:query_time:float}\s+Lock_time:\s+%{NUMBER:lock_time:float}\s+Rows_sent:\s+%{NUMBER:rows_sent:int}\s+Rows_examined:\s+%{NUMBER:rows_examined:int}\n/

[PARSER]
Name mysql_attribues
Format regex
Regex /(?m)^\s+SET\s+timestamp=%{NUMBER:timestamp};\n\s*(?<sql>(?<action>\w+)([\w.*\W.*])*;)\s*$/