log4net.redis+logstash+kibana+elasticsearch+redis 实现日志系统

前端时间写了个随笔 log4net.NoSql +ElasticSearch 实现日志记录 ，因项目原因需要把日志根java平台的同事集成采用logstash+kibana+elasticsearch+redis结构实现日志统计分析，所以需要一个将log4net日志输出到redis的组件。没有找到现成的，就自己动手了。参考了 log4net.NoSql 的代码。

redis的C#客户端使用了 ServiceStackRedis，json序列化使用 RestSharp。代码结构如下：

 

JsonLayout.cs代码：

  public class JsonLayout : LayoutSkeleton
    {
        public readonly string HostName;
        private readonly ITextTransform _transform;

        public string LogType { get; set; }
        public string AppName { get; set; }

        public JsonLayout()
            : base()
        {
            HostName = Dns.GetHostName();
            _transform = TextTransform.Instance;
        }

        public override string ContentType
        {
            get { return "application/json"; }
        }

        public override void ActivateOptions()
        {
            //nothing to do here
        }

        public override void Format(TextWriter writer, LoggingEvent loggingEvent)
        {
            var info = loggingEvent.LocationInformation;
            var loggingEventJson = _transform.Serialize(new JsonLogMessage
                {
                    class_method = info.MethodName,
                    class_name = info.ClassName,
                    host_ip = HostName,
                    line_number = info.LineNumber,
                    log_level = loggingEvent.Level.DisplayName,
                    log_message = loggingEvent.MessageObject.ToString(),
                    exception = BuildExceptionMessage(loggingEvent),
                    log_time = loggingEvent.TimeStamp.ToString("yyyy-MM-dd HH:mm:ss"),
                    logger_name = loggingEvent.LoggerName,
                    run_time = "0", //目前获取不到
                    thread_name = loggingEvent.ThreadName,
                    host = HostName,
                    log_type = this.LogType,
                    app_name = this.AppName,
                    path = ""
                });

            writer.Write(loggingEventJson);
        }

        private JsonLogException BuildExceptionMessage(LoggingEvent loggingEvent)
        {
            if (loggingEvent.ExceptionObject == null)
                return new JsonLogException { exception_class = "", exception_message = "", exception_stacktrace = "" };

            var exception = loggingEvent.ExceptionObject;
            return
                new JsonLogException
                {
                    exception_class = exception.Source,
                    exception_message = exception.Message,
                    exception_stacktrace = exception.StackTrace
                };
        }
    }

RedisAppender.cs

 public class RedisAppender : AppenderSkeleton
    {
        private volatile PooledRedisClient _pooledRedisClient;
        private readonly object _padlock = new object();

        public string Host { get; set; }
        public string Port { get; set; }
        public string ListId { get; set; }
        public string MaxPoolSize { get; set; }

        public RedisAppender()
        {
        }

        private PooledRedisClient Client
        {
            get
            {
                if (_pooledRedisClient != null) return _pooledRedisClient;

                lock (_padlock)
                {
                    if (_pooledRedisClient == null)
                    {
                        _pooledRedisClient = new PooledRedisClient(this.Host, int.Parse(this.Port), int.Parse(this.MaxPoolSize));
                    }
                }
                return _pooledRedisClient;
            }
        }

        protected override void Append(LoggingEvent loggingEvent)
        {
            var sb = new StringBuilder();
            var writer = new StringWriter(sb);
            base.Layout.Format(writer, loggingEvent);
            this.Client.AddItemToListAsync(ListId, writer.ToString());
        }
    }

PooledRedisClient.cs

  public class PooledRedisClient
    {
        private readonly string _baseUri;
        private readonly PooledRedisClientManager _clientManager;
        /// <summary>
        /// 实例化连接池客户端
        /// </summary>
        /// <param name="host"></param>
        /// <param name="port"></param>
        /// <param name="maxPoolSize">默认值10</param>
        public PooledRedisClient(string host, int port, int maxPoolSize = 10)
        {
            _baseUri = string.Format("{0}:{1}", host, port);
            var config = new RedisClientManagerConfig();
            config.MaxReadPoolSize = maxPoolSize;
            config.MaxWritePoolSize = maxPoolSize;
            _clientManager = new PooledRedisClientManager(new string[] { _baseUri }, new string[] { _baseUri }, config);
        }
        /// <summary>
        /// 异步记录
        /// </summary>
        /// <param name="listId"></param>
        /// <param name="log"></param>
        public void AddItemToListAsync(string listId, string log)
        {
            //使用Task任务异步执行
            var task = new Task(() =>
            {
                try
                {
                    using (var clinet = _clientManager.GetClient())
                    {
                        clinet.AddItemToList(listId, log);
                    }
                }
                catch (Exception ex)
                {
                    Trace.WriteLine(ex.Message);
                }
            });
            task.Start();
        }

    }

在RedisClient连接时用了连接池，写日志时用Task做了异步。这种写法不知道会不会存在问题？！

配置文件

<?xml version="1.0"?>
<log4net>
  <!--Redis日志记录-->
  <appender name="redisAppender" type="log4net.Redis.Appender.RedisAppender, log4net.Redis">
    <Host value="192.168.0.10" />
    <Port value="6379" />
    <MaxPoolSize value="500" />
    <ListId value="logstash" />
    <layout type="log4net.Redis.Layout.JsonLayout,log4net.Redis" >
      <LogType value="iis_log" />
      <AppName value="Demo" />
    </layout>
  </appender>
  <root>
    <level value="ALL" />
    <appender-ref ref="redisAppender" />
  </root>
</log4net>

Layout和Appender里的属性配置和代码里属性配置名称一致，Log4net的框架就可以读取配置数据反射到实体属性上。

代码下载地址：

http://download.csdn.net/detail/zbl131/7702673

 

参考文章：

1.用Kibana和logstash快速搭建实时日志查询、收集与分析系统

http://storysky.blog.51cto.com/628458/1158707/

2.使用ServiceStackRedis链接Redis简介

http://www.cnblogs.com/daizhj/archive/2011/02/17/1956860.html

3.ServiceStack.Redis的问题与修正

http://blog.csdn.net/susubuhui/article/details/8930417

4.对ServiceStack.Redis的连接池进行故障转移改造

http://www.cnblogs.com/smark/archive/2013/05/24/3096488.html