ASP .NET Core 使用 Serilog记录日志并输出至ElasticSearch

Serilog输出ES

Elastic (ELK) Stack的服务端部署参考这篇博客，版本都是当前最新版本8.5.0

Serilog 相关文档参考这篇博客

新建一个ASP.NET Core Web项目，添加以下Neget包

Serilog.AspNetCore
Serilog.Extensions.Logging
Serilog.Sinks.Elasticsearch

Program.cs添加Serilog并添加ES相关配置

using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
using Serilog;
using Serilog.Events;
using Serilog.Sinks.Elasticsearch;
using System;

namespace WebApplication1
{
    public class Program
    {
        public static void Main(string[] args)
        {
            CreateHostBuilder(args).Build().Run();
        }

        public static IHostBuilder CreateHostBuilder(string[] args) =>
            Host.CreateDefaultBuilder(args)
                .ConfigureWebHostDefaults(webBuilder =>
                {
                    webBuilder.UseStartup<Startup>();
                }).UseSerilog((hostingContext, loggerConfiguration) =>
                {
                    string environment = "Development";
                    loggerConfiguration
                   .ReadFrom.Configuration(hostingContext.Configuration)
                   .Enrich.FromLogContext()
                   //过滤Net Core系统日志
                   .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
                   .WriteTo.Console()
                   .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://192.168.1.5:9200/"))
                   {
                       //OverwriteTemplate和TypeName一定要加，不然ES8无法写入日志
                       
                       IndexFormat = "muxue-{0:yyyy.MM.dd}",
                       //IndexFormat = $"{Assembly.GetExecutingAssembly().GetName().Name.ToLower().Replace(".", "-")}-{environment?.ToLower().Replace(".", "-")}-{DateTime.UtcNow:yyyy-MM}",
                       AutoRegisterTemplate = true,
                       OverwriteTemplate = true,
                       //TemplateName = "",
                       FailureCallback = e => Console.WriteLine("Unable to submit event " + e.MessageTemplate),
                       AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
                       TypeName = null,
                       MinimumLogEventLevel = LogEventLevel.Verbose,
                       EmitEventFailure = EmitEventFailureHandling.RaiseCallback,
                       BatchAction = ElasticOpType.Create,
                       BatchPostingLimit = 50,//一次批量发送日志数量，默认50
                       ModifyConnectionSettings =
                            conn =>
                            {
                                //conn.BasicAuthentication("elastic", "123456");
                                conn.ServerCertificateValidationCallback((source, certificate, chain, sslPolicyErrors) => false);
                                return conn;
                            }
                   });
                });
    }
}

字段	备注
ElasticsearchSinkOptions(URL)	ES访问地址
IndexFormat	ES索引，可自定义。{0:yyyy.MM.dd} 表示日期
AutoRegisterTemplate	是否ES日志自动注册一个索引模板。
OverwriteTemplate	是否覆盖ES日志默认模板，ES8默认不支持写入，需要加此配置
TemplateName	ES日志模板
EmitEventFailure	设置了当失败时调用FailureCallback
FailureCallback	日志发送失败触发事件
AutoRegisterTemplateVersion	ES模板版本
MinimumLogEventLevel	最低日志等级
TypeName	ES8默认不支持写入，需要加此配置
ModifyConnectionSettings	ES认证，用户登录+是否开启SSL(Https)
BatchPostingLimit	一次性发送日志数量

WeatherForecastController.cs写入日志并访问API

using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
using System;

namespace WebApplication1.Controllers
{
    [ApiController]
    [Route("[controller]")]
    public class WeatherForecastController : ControllerBase
    {
        private readonly ILogger<WeatherForecastController> _logger;

        public WeatherForecastController(ILogger<WeatherForecastController> logger)
        {
            _logger = logger;
        }

        [HttpGet]
        public string Get()
        {
            _logger.LogInformation("我爱中国");
            _logger.LogInformation(Guid.NewGuid().ToString());
            return Guid.NewGuid().ToString();
        }
    }
}

查看日志

Serilog+Logstash输出ES

日志文件模板参考：此文档日志配置汇总模块
Logstashconf配置。要注意的是索引名称必须全部小写。

input {
    file {
        path => "/usr/share/logstash/logs/*.log"
        start_position => "beginning"
	codec => json {
	  charset => ["UTF-8"]
	  }
	#扫描间隔时间，默认是1s，建议5s
        stat_interval => "5"
    }
}

output {
   elasticsearch {
        hosts => ["elasticsearch:9200"]
	#索引名称必须全部小写
        index => "%{systemName}-%{serviceName}-%{logType}-%{+YYYY.MM.dd}"
   }
}

Serilog+Logstash Http输出ES

添加以下Neget包

Serilog.AspNetCore
Serilog.Sinks.Http

Program.cs添加Serilog并添加ES相关配置

 public static IHostBuilder CreateHostBuilder(string[] args) =>
            Host.CreateDefaultBuilder(args)
                .ConfigureWebHostDefaults(webBuilder =>
                {
                    webBuilder.UseStartup<Startup>();
                }).UseSerilog((hostingContext, loggerConfiguration) =>
                {
                    loggerConfiguration
                   .Enrich.FromLogContext()

                   //过滤Net Core系统日志
                   .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)

                   //发送至logstash地址；这里不用http方法是因为官方文档描述此方法在网路延宕重启时并不会保留数据
                   //会生成两个Buffer存储文件，通过内容发送请求
                   .WriteTo.DurableHttpUsingFileSizeRolledBuffers("http://192.168.1.5:9650")
                   .WriteTo.Console();
                   
                });

/usr/share/logstash/pipeline新建net6.conf输入以下内容并重启logstash，启动站点写入Serilog日志

input {
   http {
        host => "0.0.0.0"
        port => 9650
        additional_codecs => {"application/json"=>"json"}
        codec => "plain"
        threads => 1
        ssl => false
    }
}

output {
  elasticsearch {
    hosts => ["192.168.1.5:9200"]
    user => "elastic"
    password => "123546"
    #索引名
    index => "net6api-%{+YYYY.MM.dd}"
 }
}

查看日志

Serilog+RabbitMQ+Logstash输出ES

RabbitMQ部署及使用请参考这篇博客

安装以下Neget包

Serilog.AspNetCore
Serilog.Sinks.RabbitMQ

Program.cs添加Serilog并添加RabbitMQ相关配置

using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
using Serilog;
using Serilog.Events;

namespace WebApplication1
{
    public class Program
    {
        public static void Main(string[] args)
        {


            CreateHostBuilder(args).Build().Run();
        }

        public static IHostBuilder CreateHostBuilder(string[] args) =>
            Host.CreateDefaultBuilder(args)
                .ConfigureWebHostDefaults(webBuilder =>
                {
                    webBuilder.UseStartup<Startup>();
                }).UseSerilog((hostingContext, loggerConfiguration) =>
                {
                    loggerConfiguration
                   .Enrich.FromLogContext()
                   //过滤Net Core系统日志
                   .MinimumLevel.Override("Microsoft", LogEventLevel.Warning)
                   .WriteTo.RabbitMQ((clientConfiguration, sinkConfiguration) => {
                        clientConfiguration.Hostnames.Add("192.168.1.5");
                        clientConfiguration.Username = "guest";
                        clientConfiguration.Password = "guest";
                        clientConfiguration.Exchange = "rqlogstashExchange";
                        clientConfiguration.ExchangeType = RabbitMQ.Client.ExchangeType.Direct;
                        clientConfiguration.DeliveryMode = Serilog.Sinks.RabbitMQ.RabbitMQDeliveryMode.Durable;
                        clientConfiguration.RouteKey = "rqlogstash";
                        clientConfiguration.Port = 5672;
                        sinkConfiguration.TextFormatter = new Serilog.Formatting.Json.JsonFormatter();
                     })
                   .WriteTo.Console();
                   
                });
    }
}

或者RabbitMQ生产者直接传递消息。安装Neget包RabbitMQ.Client

using System.IO;
using System.Linq;
using RabbitMQ.Client;
using System;

namespace ConsoleApp1
{
    class Program
    {
        static void Main(string[] args)
        {
            Console.WriteLine("Hello, World! 生产者");

            var factory = new ConnectionFactory()       // 创建连接工厂对象
            {
                HostName = "192.168.1.5",
                Port = 5672,
                UserName = "guest",
                Password = "guest"
            };
            var connection = factory.CreateConnection();    // 创建连接对象
            var channel = connection.CreateModel();         // 创建连接会话对象


            string routeKey = "rqlogstash";

            string exchangeName = "rqlogstashExchange";

            channel.ExchangeDeclare(exchange: exchangeName, type: ExchangeType.Direct);

            string str;
            do
            {
                Console.WriteLine("发送内容：");
                str = Console.ReadLine()!;

                byte[] body = System.Text.Encoding.UTF8.GetBytes(str); // 消息内容

                channel.BasicPublish(exchangeName, routeKey, null, body); // 发送消息
            } while (str.Trim().ToLower() != "exit");

            channel.Close();
            connection.Close();
        }
    }
}

LogLogstash收集日志/usr/share/logstash/pipeline/rabbitmq.conf配置文件

input {
  rabbitmq {
    host => "192.168.1.5"
	# 虚拟机Host
	#vhost => "Ocsa_Cap" 
    port => 5672
    user => "guest"
    password => "guest"
    queue => "logstash"
    key => "rqlogstash"
    exchange => "rqlogstashExchange"
	#持久化跟队列配置一致
    durable => true 
	#格式
	codec => "plain"       
  }
}


filter {
  grok {
    match => {"Timestamp" => "%{TIMESTAMP_ISO8601:ctime}"}
    add_field => ["create_time","%{@timestamp}"]
  }
  date {
    match => ["ctime","yyyy-MM-dd HH:mm:ss.SSS","ISO8601"]
    target => "@timestamp"
  }
  mutate {
    remove_field => ["@version","Properties","Timestamp","ctime"]
    rename => {"MessageTemplate" => "message"}
    rename => {"Level" => "level"}
  }
  ruby {
    code => "event.set('create_time',event.get('@timestamp').time.localtime)"
  }
}

output {
  elasticsearch {
    hosts => ["192.168.1.5:9200"]
    index => "rabbit-%{+YYYYMMdd}"
    user => "elastic"
    password => "wHUIbdI4wD66avRX6mc2"

  }
}

查看日志