Logstash（一）模拟生成log日志，纯java

一、目录

二、代码

1.App启动项

package cn.kgc.kb08.ls;

import cn.kgc.kb08.ls.make.LogService;

public class App {
  public static void main(String[] args) {
    (new LogService("/root/mylog.log")).write(args); // 可以传入多个参数 1.进入write
  }
}

2.logservice

package cn.kgc.kb08.ls.make;

import cn.kgc.kb08.ls.dao.LogDao;
import java.text.MessageFormat;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class LogService {
  private String path;
  
  private Random rand = new Random();
  
  private ExecutorService pool = Executors.newFixedThreadPool(50);
  
  private String[] eventName = new String[] { "btn_page", "btn_search", "btn_add_cart", "btn_pay" };
  
  private String[] browser = new String[] { "firefox", "chrome", "ie", "360" };
  
  private String[] timeZone = new String[] { "UTC", "GMT" };
  
  public LogService(String path) {
    this.path = path;
  }
  
  private String log(String type) { // 生成一条什么type的记录
    Object[] time = { String.valueOf(2010 + this.rand.nextInt(11)), String.valueOf(1 + this.rand.nextInt(12)), String.valueOf(1 + this.rand.nextInt(30)) };
    String eventTime = MessageFormat.format("{0}-{1}-{2}", time);
    Object[] info = { Integer.valueOf(1 + this.rand.nextInt(100)), String.valueOf(1 + this.rand.nextInt(2000)), this.eventName[this.rand.nextInt(this.eventName.length)], eventTime, this.browser[this.rand.nextInt(this.browser.length)], this.timeZone[this.rand.nextInt(this.timeZone.length)] };
    return type.equals("0") ? 
      MessageFormat.format("userId:{0}|clickCount:{1}|eventName:{2}|clickTime:{3}|browser:{4}|timeZone:{5}", info) : 
      MessageFormat.format("{0}|{1}|{2}|{3}|{4}|{5}", info);
  }
  
  public void write(String[] args) {   // 2. 处理type和记录数量num：生成一条type的记录并写入path的文件中，该操作执行num次数
    String type = args[0];
    int times = Integer.parseInt(args[1]);
    for (int i = 0; i < times; i++)
      this.pool.submit((Runnable)new LogDao(this.path, log(type))); // 先赋值，再写入一条数据
    this.pool.shutdownNow();
  }
}

3.logDao

package cn.kgc.kb08.ls.dao;

import java.io.IOException;
import java.io.RandomAccessFile;

public class LogDao implements Runnable {
  private String msg;
  
  private String path;
  
  private static synchronized void info(String path, String msg) { // 将一条记录添加到path文件的末尾
    RandomAccessFile rand = null;
    try {
      rand = new RandomAccessFile(path, "rw");
      rand.seek(rand.length());
      rand.writeBytes(msg + "\r\n");
    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      if (null != rand)
        try {
          rand.close();
        } catch (IOException e) {
          e.printStackTrace();
        }  
    } 
  }
  
  public LogDao(String msg, String path) { // 先赋值，再执行info()的操作
    this.msg = msg;
    this.path = path;
  }
  
  public void run() {
    info(this.msg, this.path);
  }
}

 

启动项是App。传入两个参数

 

打成jar包时要注意：jar包用rar打开，根目录下加上 META-INF/MANIFEST.MF文件指定启动项

命令：java -jar lsdemo.jar 0 10000

结果：/root/下生成 mylog.log

四、使用虚拟日志作为input，通过logstash这个管道

cd /root/ 

touch logstash.log

vi logstash.log

input{
 // 从mylog.log中拿数据
    file{
        path => "/root/mylog.log"
        start_position => "beginning"
         sincedb_path => "/dev/null"
        type => "go"
    }
}
// 过滤加工log中的数据：使用正则匹配拿到的数据如下：
// userId:32|clickCount:890|eventName:btn_add_cart|clickTime:2018-12-16|browser:ie|timeZone:UTC 
filter{
       if[type] == "go"{
        grok{
          //错了，文末订正  match => {"message" => "%{NUMBER:userid}\|%{NUMBER:clickCount}\|%{WORD:eventName}\|%{TIMESTAMP_ISO8601:clickTime}\|%{WORD:browser}\|%{WORD:timeZone}"}
            remove_field =>["message"]
        }
    }
}
// 用go的type来输出给ES
output{
    if[type] == "go"{
                elasticsearch{
                        hosts => ["http://192.168.203.132:9200","http://192.168.203.142:9200","http://192.168.203.152:9200"]
                        index => "logstashdb"
                        document_type => "behaviour"
                }
        }
}

五、启动logstash

cd到 logstash的bin目录下，命令如下：

nohup ./logstash -f ~/logstash.log （-f 表示用一个file作为输入）

 ------------------------------补充------------------------------

1.正则的写法

 

 2. 输出前先在控制台测试，语句如下：