logstash多数据源配置

　　原型为canal同步MySQL数据库数据流程如下：

　　　　canal --> kafka --> logstash --> es 

　　　　

input {
kafka {
bootstrap_servers => "192.168.169.18:9093"
consumer_threads => "5"
auto_offset_reset => "latest"
topics => ["test_community_own","test_community_attention","test_community_community"]
group_id=> "logstash_test_community"
client_id => "logstash_test_community"
decorate_events => true
codec => "json"
}
}
filter {
if [@metadata][kafka][topic] == "test_community_own" {
json{
source => "message"
tag_on_failure => "logtxt"
}
split{
field => "[data]"
}

mutate{
add_field => {
"doc_id" => "%{[data][id]}"
"video_id" => "%{[data][video_id]}"
"user_id" => "%{[data][user_id]}"
"create_time" => "%{[data][create_time]}"


}
remove_field => ["mysqlType", "pkNames", "sqlType", "@version", "isDdl", "type", "table", "@timestamp", "ts", "old", "sql", "database", "es", "id", "data"]
}
mutate{
convert => {
"doc_id" => "integer"
"video_id" => "string"
"user_id" => "string"
"create_time" => "integer"
}
}
}
else if [@metadata][kafka][topic] == "test_community_attention" {
json{
source => "message"
tag_on_failure => "logtxt"
}
split{
field => "[data]"
}
mutate{
add_field => {
"doc_id" => "%{[data][id]}"
"user_id" => "%{[data][user_id]}"
"video_id" => "%{[data][video_id]}"
"difficulty" => "%{[data][difficulty]}"
"create_time" => "%{[data][create_time]}"
"update_time" => "%{[data][update_time]}"
}
remove_field => ["mysqlType", "pkNames", "sqlType", "@version", "isDdl", "type", "table", "@timestamp", "ts", "old", "sql", "database", "es", "id", "data"]
}

mutate{
convert => {
"doc_id" => "integer"
"user_id" => "string"
"video_id" => "string"
"difficulty" => "integer"
"create_time" => "integer"
"update_time" => "integer"
}
}
}
else if [@metadata][kafka][topic] == "test_community_community" {
json{
source => "message"
tag_on_failure => "logtxt"
}
split{
field => "[data]"
}
mutate{
add_field => {
"doc_id" => "%{[data][id]}"
"user_id" => "%{[data][user_id]}"
"video_id" => "%{[data][video_id]}"
"difficulty" => "%{[data][difficulty]}"
"secret" => "%{[data][secret]}"
"is_publish" => "%{[data][is_publish]}"
"is_work" => "%{[data][is_work]}"
"media_type" => "%{[data][media_type]}"
"shoot_type" => "%{[data][shoot_type]}"
"create_time" => "%{[data][create_time]}"
"update_time" => "%{[data][update_time]}"
"pool" => "%{[data][pool]}"
"likes" => "%{[data][likes]}"
"comment" => "%{[data][comment]}"
"click" => "%{[data][click]}"
"full_play" => "%{[data][full_play]}"
"forward" => "%{[data][forward]}"
"exposure" => "%{[data][exposure]}"
"score" => "%{[data][score]}"

}
remove_field => ["mysqlType", "pkNames", "sqlType", "@version", "isDdl", "type", "table", "@timestamp", "ts", "old", "sql", "database", "es", "id", "data"]
}

mutate{
convert => {
"doc_id" => "integer"
"user_id" => "string"
"video_id" => "string"
"difficulty" => "integer"
"secret" => "integer"
"is_publish" => "integer"
"is_work" => "integer"
"media_type" => "integer"
"shoot_type" => "integer"
"create_time" => "integer"
"update_time" => "integer"
"pool" => "integer"
"likes" => "integer"
"comment" => "integer"
"click" => "integer"
"full_play" => "integer"
"forward" => "integer"
"exposure" => "integer"
"score" => "float"
}
}
}
}

output {
if [@metadata][kafka][topic] == "test_community_own" {
elasticsearch {
hosts => ["http://192.168.169.15:9201"]
index => "test_community_own-yy"
document_type => "test_community_own-yy_info"
document_id => "%{doc_id}"
}
}
else if [@metadata][kafka][topic] == "test_community_attention" {
elasticsearch {
hosts => ["http://192.168.169.15:9201"]
index => "test_community_attention-yy"
document_type => "test_community_attention-yy_info"
document_id => "%{doc_id}"
}
}
else if [@metadata][kafka][topic] == "test_community_community" {
elasticsearch {
hosts => ["http://192.168.169.15:9201"]
index => "test_community_community-yy"
document_type => "test_community_community-yy_info"
document_id => "%{doc_id}"
}
}

}

 以上配置为本人自定义配置，且本文为原创，若是有相关问题请留言，转载请标明出处。