rsyslog+loki日志服务器搭建
一、更新服务器，关闭selinux
yum update -y
vim /etc/selinux/config
SELINUX=disabled
二、搭建rsyslog
1、创建日志存放目录及权限
mkdir /var/log/network
chmod 700 -R /var/log/network/
chown -R promtail:promtail /var/log/network/
2、优化日志格式和存放目录
vim /etc/rsyslog.d/default.conf
#### GLOBAL DIRECTIVES ####
# $FileGroup - Set the group for dynaFiles newly created
# ----------
$FileGroup promtail #自动生成的日志文件修改其属组
# $FileOwner - Set the file owner for dynaFiles newly created.
# ----------
$FileOwner promtail #自动生成的日志文件修改其属主
# Don't log private authentication messages!
*.info;mail.none;authpriv.none;cron.none;local5.none;local1.none;local2.none;local3.none;local4.none;local6.none;local7.none;kern.none /var/log/messages
#### GLOBAL DIRECTIVES ####
# Use default timestamp format # 日志消息格式自定义的格式
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$template myFormat,"%timegenerated% %fromhost-ip% %msg%\n"
$ActionFileDefaultTemplate myFormat #默认日志格
$template VPNRemoteLogsSystemformat,"%timegenerated:1:10:date-rfc3339% %timereported:12:19:date-rfc3339% User:%msg:F,32:4% Src:%msg:F,32:13% dst:%msg:F,32:17%\n"
$template VPNRemoteLogsSessionformat,"%timereported:12:19:date-rfc3339% %msg:F,59:4% %msg:F,59:5% %msg:F,59:6% %msg:F,59:7% %msg:F,59:8% %msg:F,59:9% %msg:F,59:10% %msg:F,59:11% %msg:F,59:12%\n"
# 根据客户端的IP或者日志类型不同存放主机日志在不同目录，rsyslog需要手动创建
$template RemoteLogs,"/var/log/network/%fromhost-ip%/%$YEAR%-%$MONTH%-%$DAY%.log"
$template VPNRemoteLogsSystem,"/var/log/network/%fromhost-ip%/System-%$YEAR%-%$MONTH%-%$DAY%.log"
$template VPNRemoteLogsSession,"/var/log/network/%fromhost-ip%/Session-%$YEAR%-%$MONTH%-%$DAY%.log"
#:msg,contains,"VsysId:1" ?VPNRemoteLogsSession
#:msg,contains,"10SSLVPN/6/SSLVPN_IP_RESOURCE_PERMIT" ?VPNRemoteLogsSystem
#匹配相应的日志调用相关的日志格式、存放路径以及日志文件的权限
:msg,contains,"10SSLVPN/6/SSLVPN_IP_RESOURCE_PERMIT" action(type="omfile" FileGroup="promtail" FileOwner="promtail" FileCreateMode="0600" dirCreateMode="0600" dynaFile="VPNRemoteLogsSystem" template="VPNRemo
teLogsSystemformat")
:msg,contains,"VsysId:1" action(type="omfile" FileGroup="promtail" FileOwner="promtail" FileCreateMode="0600" dirCreateMode="0600" dynaFile="VPNRemoteLogsSession" template="VPNRemoteLogsSessionformat")
# 排除本地相关IP日志记录，针对有日志分割的文件，没有分割的文件不会存储在message
if $fromhost-ip != '127.0.0.1' and $fromhost-ip != '10.122.8.10' then ?RemoteLogs
# 忽略之前所有的日志，远程主机日志记录完之后不再继续往下记录
& ~
3、重启服务
systemctl restart rsyslog
三、搭建promtail
1、下载二进制安装包
wget https://github.com/grafana/loki/releases/download/v2.7.2/promtail-linux-amd64.zip
unzip -d /usr/local/bin/ promtail-linux-amd64.zip #解压到指定目录
mkdir -pv /etc/promtail #创建配置文件目录
2、下载配置文件模板
wget https://github.com/grafana/loki/blob/main/clients/cmd/promtail/promtail-local-config.yaml
mv promtail-local-config.yaml config-promtail.yml
3、编辑配置文件
vim /etc/promtail/config-promtail.yml
server: http_listen_port: 9080 grpc_listen_port: 0 positions: filename: /tmp/positions.yaml clients: - url: http://localhost:3100/loki/api/v1/push scrape_configs: - job_name: syslog static_configs: - targets: - localhost labels: job: switch-core location: YiKu vendor: huawei hostname: 192.168.101.1 __path__: /var/log/network/192.168.101.1/*log - targets: - localhost labels: job: switch-core-wifi location: shenzhen # 设备的机房或者所在的位置 vendor: huawei # 品牌 hostname: 192.168.101.61 # 主机名 __path__: /var/log/network/192.168.101.61/*log # 日志的路径 ~
4、promtail开机自启动
vim /lib/systemd/system/promtail.service
[Unit] Description=Promtail service After=network.target [Service] Type=simple User=promtail ExecStart=/usr/local/bin/promtail-linux-amd64 -config.file /etc/promtail/config-promtail.yml [Install] WantedBy=multi-user.target
useradd -r -s /sbin/nologin promtail && chmod 700 -R /etc/promtail && chown promtail:promtail -R /etc/promtail
systemctl start promtail
systemctl enable promtail
四、搭建loki
1、下载二进制安装包，调整权限
curl -O -L "https://github.com/grafana/loki/releases/download/v2.7.2/loki-linux-amd64.zip"
unzip -d /usr/local/bin/ loki-linux-amd64.zip #解压到指定目录
useradd -r -s /sbin/nologin loki #创建系统用户，设定为不能登录
mkdir -pv /etc/loki /data/loki #创建配置文件目录和数据目录
chown -R loki:loki /etc/loki /data/loki #更改该配置文件目录的属主属组
chmod -R 700 /etc/loki /data/loki #更改配置文件目录和数据目录的权限
2、下载配置文件模板
wget https://raw.githubusercontent.com/grafana/loki/master/cmd/loki/loki-local-config.yaml
mv loki-local-config.yaml config-loki.yaml
3、编辑配置文件
vim /etc/loki/config-loki.yaml
auth_enabled: false server: http_listen_port: 3100 grpc_listen_port: 9096 common: instance_addr: 192.168.77.21 #loki本机地址 path_prefix: /data/loki #定位到数据目录下 storage: filesystem: chunks_directory: /data/loki/chunks #定位到数据目录下 rules_directory: /data/loki/rules #定位到数据目录下 replication_factor: 1 ring: kvstore: store: inmemory query_range: results_cache: cache: embedded_cache: enabled: true max_size_mb: 100 schema_config: configs: - from: 2023-07-20 store: boltdb-shipper object_store: filesystem schema: v11 index: prefix: index_ period: 24h ruler: alertmanager_url: http://localhost:9093
4、loki开机自启动
[Unit] Description=Loki service After=network.target [Service] Type=simple User=loki ExecStart=/usr/local/bin/loki-linux-amd64 -config.file /etc/loki/config-loki.yaml [Install] WantedBy=multi-user.target
systemctl enable loki
systemctl start loki
systemctl status loki
五、搭建grafana
1、下载grafana rpm包，并设定开机自启
wget https://dl.grafana.com/enterprise/release/grafana-enterprise-9.2.3-1.x86_64.rpm
yum install grafana-enterprise-9.2.3-1.x86_64.rpm
systemctl daemon-reload
systemctl enable grafana-server
systemctl start grafana-server
systemctl status grafana-server
六、firewall放行相关的端口
firewall-cmd --permanent --zone=public --add-port=515/tcp --add-port=515/udp --add-port=3000/tcp --add-port=9080/tcp --add-port=9096/tcp --add-port=3100/tcp --add-port=9600/tcp --add-port=9093/tcp
firewall-cmd --reload
firewall-cmd --list-ports
日志格式化featrue
"％msg:1:2％" # 提取消息文本的前两个字符 "％msg:80:$％" # 从第80个字符开始一直截取到末尾的剩下的文本 "％msg:::drop-last-lf％" #获取日志消息的整个消息文本，并删除其最后一个换行符 "%msg:R:Inter.*--end%\n" # 正则匹配从字符Inter开始到该行结尾，--end为固定搭配 "%msg:F,32:5%\n" # 以空格为分隔符取第五列（32在ASCII码中表示空格） "%msg:F,59,1:5,12%\n" # 以;为分隔符，取第五列中的第一个字符到第十二个字符（59在ASCII码中表示分号，5表示第五列）
