elk es
能把修改过的行显示出来
[root@es2 ~]# grep -Pv "^(#|$)" /etc/elasticsearch/elasticsearch.yml
cluster.name: nsd1808
node.name: es2
network.host: 0.0.0.0
discovery.zen.ping.unicast.hosts: ["es1", "es2", "es3"]
用命令行访问
[root@es1 ~]# curl 118.144.89.240/info.php
<pre>
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> curl/7.29.0 显示客户端的设备信息
[ HTTP_REFERER] ==>
--- --- GET options is --- ---
Array
(
)
谷歌浏览器访问
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36
[ HTTP_REFERER] ==>
--- --- GET options is --- ---
Array
(
)
[root@es1 ~]# curl -A "test" http://118.144.89.240/info.php 用-A可以修改显示的客户端设备信息
<pre>
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> test
[ HTTP_REFERER] ==>
--- --- GET options is --- ---
Array
(
)
[root@es1 ~]# curl -X "GET" http://118.144.89.240/info.php
<pre>
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> curl/7.29.0
[ HTTP_REFERER] ==>
--- --- GET options is --- ---
Array
(
)
[root@es1 ~]# curl -X "DELETE" http://118.144.89.240/info.php
<pre>
[ REQUEST_METHOD] ==> DELETE
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> curl/7.29.0
[ HTTP_REFERER] ==>
--- --- DELETE options is --- ---
Array
(
)
分片数要小于等于集群的台数，副本数要少于分片数
[root@es5 ~]# cd bigdesk
[root@es5 bin]# ./plugin install file:///root/elk软件包/elasticsearch-head-master.zip
[root@es5 bin]# ./plugin install file:///root/elk软件包/elasticsearch-kopf-master.zip
[root@es5 bin]# ./plugin install file:///root/elk软件包/bigdesk-master.zip
[root@es5 bin]# ./plugin list
火狐浏览器访问
http://192.168.3.115:9200/_plugin/head/
http://192.168.3.115:9200/_plugin/kopf/
http://192.168.3.115:9200/_plugin/bigdesk/
[root@es5 ~]# curl http://192.168.3.112:9200/_cat
=^.^=
/_cat/allocation
/_cat/shards
/_cat/shards/{index}
/_cat/master
/_cat/nodes
/_cat/indices
/_cat/indices/{index}
/_cat/segments
/_cat/segments/{index}
/_cat/count
/_cat/count/{index}
/_cat/recovery
/_cat/recovery/{index}
/_cat/health
/_cat/pending_tasks
/_cat/aliases
/_cat/aliases/{alias}
/_cat/thread_pool
/_cat/plugins
/_cat/fielddata
/_cat/fielddata/{fields}
/_cat/nodeattrs
/_cat/repositories
/_cat/snapshots/{repository}
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/master
-f2FWBalRg-MC0lq-QlNUQ 192.168.3.113 192.168.3.113 es3
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/master?v
id host ip node
-f2FWBalRg-MC0lq-QlNUQ 192.168.3.113 192.168.3.113 es3
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/master?help
id | | node id
host | h | host name
ip | | ip address
node | n | node name
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/nodes
192.168.3.113 192.168.3.113 3 55 0.00 d * es3 有星号*代表master
192.168.3.114 192.168.3.114 9 53 0.00 d m es4
192.168.3.115 192.168.3.115 12 67 0.00 d m es5
192.168.3.112 192.168.3.112 10 55 0.00 d m es2
192.168.3.111 192.168.3.111 10 64 0.01 d m es1
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/indices?v
health status index pri rep docs.count docs.deleted store.size pri.store.size
green open abc 5 1 0 0 1.5kb 795b
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/shards?v
index shard prirep state docs store ip node
abc 1 p STARTED 0 159b 192.168.3.111 es1 这里p代表主，master
abc 1 r STARTED 0 159b 192.168.3.112 es2
abc 4 r STARTED 0 159b 192.168.3.113 es3
abc 4 p STARTED 0 159b 192.168.3.115 es5
abc 2 p STARTED 0 159b 192.168.3.112 es2
abc 2 r STARTED 0 159b 192.168.3.114 es4
abc 3 p STARTED 0 159b 192.168.3.114 es4
abc 3 r STARTED 0 159b 192.168.3.115 es5
abc 0 p STARTED 0 159b 192.168.3.113 es3
abc 0 r STARTED 0 159b 192.168.3.111 es1
#############################################################################################
任意一台eslastsearch虚拟机。
这里在虚拟机es5
在文本里写好以下内容后，再粘贴到命令行
curl -XPUT http://192.168.3.112:9200/nsd1808 -d '
{
"settings": {
"index": {
"number_of_shards": 5 ,
"number_of_replicas": 1
}
}
}'
这是在命令行粘贴后的结果
[root@es5 ~]# curl -XPUT http://192.168.3.112:9200/nsd1808 -d '
> {
> "settings": {
> "index": {
> "number_of_shards": 5 ,
> "number_of_replicas": 1
> }
> }
> }'
在文本里写好以下内容后，再粘贴到命令行
curl -XPUT http://192.168.3.112:9200/nsd1808/xs/1 -d '
{
"姓名": "张三",
"爱好": "吃",
"阶段": 1,
"年龄": 20
}'
这是在命令行粘贴后的结果
[root@es5 ~]# curl -XPUT http://192.168.3.112:9200/nsd1808/xs/1 -d '
> {
> "姓名": "张三",
> "爱好": "吃",
> "阶段": 1,
> "年龄": 20
> }'
显示如下：
{"_index":"nsd1808","_type":"xs","_id":"1","_version":1,"_shards":{"total":2,"successful":2,"failed":0},"created":true}[root@es5 ~]#
在文本里写好以下内容后，再粘贴到命令行
curl -XPOST http://192.168.3.112:9200/nsd1807/xs/1/_update -d '
{
"doc":{
"爱好": "吃喝睡"
}
}'
这是在命令行粘贴后的结果
[root@es5 ~]# curl -XPOST http://192.168.3.112:9200/nsd1807/xs/1/_update -d '
> {
> "doc":{
> "爱好": "吃喝睡"
> }
> }'
显示如下：
{"_index":"nsd1807","_type":"xs","_id":"1","_version":2,"_shards":{"total":2,"successful":2,"failed":0}}[root@es5 ~]#
[root@es5 ~]# curl -XDELETE http://192.168.3.112:9200/nsd1808
显示如下：
{"acknowledged":true}
[root@es5 ~]# curl -XDELETE http://192.168.3.112:9200/*
显示如下：
{"acknowledged":true}
#############################################################################################
虚拟机kibana
[root@kibana json]# ls
accounts.json.gz logs.jsonl.gz shakespeare.json.gz
[root@kibana json]# for i in * ; do gzip -d $i ;done 解包
[root@kibana json]# ls
accounts.json logs.jsonl shakespeare.json
[root@kibana json]# curl -XPOST http://192.168.3.111:9200/_bulk --data-binary @shakespeare.json
[root@kibana json]# curl -XPOST http://192.168.3.111:9200/_bulk --data-binary @logs.jsonl
索引是aa，类型是bb，必须导入索引和类型，没有索引，要加上
[root@kibana json]# curl -XPOST http://192.168.3.111:9200/aa/bb/_bulk --data-binary @accounts.json
#############################################################################################
虚拟机kibana
在文本里写好以下内容后，再粘贴到命令行
curl -XGET http://192.168.3.111:9200/_mget?pretty -d '
{
"docs" : [
{
"_index" : "aa",
"_type" : "bb",
"_id" : "190"
},
{
"_index" : "shakespeare",
"_type" : "line",
"_id" : "24"
},
{
"_index" : "logstash-2015.05.18",
"_type" : "log",
"_id" : "AWfoHycao9bttkMidMF9"
}
]
}'
这是在命令行粘贴后的结果
[root@kibana json]# curl -XGET http://192.168.3.111:9200/_mget?pretty -d '
> {
> "docs" : [
> {
> "_index" : "aa",
> "_type" : "bb",
> "_id" : "190"
> },
> {
> "_index" : "shakespeare",
> "_type" : "line",
> "_id" : "24"
> },
> {
> "_index" : "logstash-2015.05.18",
> "_type" : "log",
> "_id" : "AWfoHycao9bttkMidMF9"
> }
> ]
> }'
显示如下：
{
"docs" : [ {
"_index" : "aa",
"_type" : "bb",
"_id" : "190",
"_version" : 1,
"found" : true,
"_source" : {
"account_number" : 190,
"balance" : 3150,
"firstname" : "Blake",
"lastname" : "Davidson",
"age" : 30,
"gender" : "F",
"address" : "636 Diamond Street",
"employer" : "Quantasis",
"email" : "blakedavidson@quantasis.com",
"city" : "Crumpler",
"state" : "KY"
}
}, {
"_index" : "shakespeare",
"_type" : "line",
"_id" : "24",
"_version" : 1,
"found" : true,
"_source" : {
"line_id" : 25,
"play_name" : "Henry IV",
"speech_number" : 1,
"line_number" : "1.1.22",
"speaker" : "KING HENRY IV",
"text_entry" : "Forthwith a power of English shall we levy;"
}
}, {
"_index" : "logstash-2015.05.18",
"_type" : "log",
"_id" : "AWfoHycao9bttkMidMF9",
"_version" : 1,
"found" : true,
"_source" : {
"@timestamp" : "2015-05-18T19:25:45.051Z",
"ip" : "210.155.113.62",
"extension" : "jpg",
"response" : "200",
"geo" : {
"coordinates" : {
"lat" : 48.47088889,
"lon" : -122.4208611
},
"src" : "CO",
"dest" : "IN",
"srcdest" : "CO:IN"
},
"@tags" : [ "success", "info" ],
"utc_time" : "2015-05-18T19:25:45.051Z",
"referer" : "http://www.slate.com/success/duane-g-carey",
"agent" : "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24",
"clientip" : "210.155.113.62",
"bytes" : 5630,
"host" : "media-for-the-masses.theacademyofperformingartsandscience.org",
"request" : "/uploads/owen-garriott.jpg",
"url" : "https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/owen-garriott.jpg",
"@message" : "210.155.113.62 - - [2015-05-18T19:25:45.051Z] \"GET /uploads/owen-garriott.jpg HTTP/1.1\" 200 5630 \"-\" \"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\"",
"spaces" : "this is a thing with lots of spaces wwwwoooooo",
"xss" : "<script>console.log(\"xss\")</script>",
"headings" : [ "<h3>lawrence-j-delucas</h5>", "http://www.slate.com/success/robert-s-kimbrough" ],
"links" : [ "charles-camarda@www.slate.com", "http://nytimes.com/info/mark-brown", "www.twitter.com" ],
"relatedContent" : [ {
"url" : "http://www.laweekly.com/news/tim-biskup-mural-on-la-brea-2368038",
"og:type" : "article",
"og:title" : "Tim Biskup Mural on La Brea",
"og:description" : "Yeah, it's actually an ad for Helio... Kind of. There's a small Helio logo off to the left side. But for the most part it's just a cool huge original pa...",
"og:url" : "http://www.laweekly.com/news/tim-biskup-mural-on-la-brea-2368038",
"article:published_time" : "2007-12-20T07:16:16-08:00",
"article:modified_time" : "2014-10-28T14:59:54-07:00",
"article:section" : "News",
"og:image" : "http://IMAGES1.laweekly.com/imager/tim-biskup-mural-on-la-brea/u/original/2430352/img_1746.jpg",
"og:image:height" : "360",
"og:image:width" : "480",
"og:site_name" : "LA Weekly",
"twitter:title" : "Tim Biskup Mural on La Brea",
"twitter:description" : "Yeah, it's actually an ad for Helio... Kind of. There's a small Helio logo off to the left side. But for the most part it's just a cool huge original pa...",
"twitter:card" : "summary",
"twitter:image" : "http://IMAGES1.laweekly.com/imager/tim-biskup-mural-on-la-brea/u/original/2430352/img_1746.jpg",
"twitter:site" : "@laweekly"
}, {
"url" : "http://www.laweekly.com/music/friday-video-ozark-jubilee-2400136",
"og:type" : "article",
"og:title" : "Friday Video: Ozark Jubilee",
"og:description" : "The Ozark Jubilee: Li'l Liza Jane and more clips from the 1950s TV show",
"og:url" : "http://www.laweekly.com/music/friday-video-ozark-jubilee-2400136",
"article:published_time" : "2007-07-20T12:40:45-07:00",
"article:modified_time" : "2014-11-27T07:07:25-08:00",
"article:section" : "Music",
"og:site_name" : "LA Weekly",
"twitter:title" : "Friday Video: Ozark Jubilee",
"twitter:description" : "The Ozark Jubilee: Li'l Liza Jane and more clips from the 1950s TV show",
"twitter:card" : "summary",
"twitter:site" : "@laweekly"
}, {
"url" : "http://www.laweekly.com/arts/rock-with-me-2374144",
"og:type" : "article",
"og:title" : "Rock With Me",
"og:description" : ""How many boulders can we possibly have to climb over?" I chuckled to my friend Julie, when she warned me that her boyfriend David's friend Ben had aske...",
"og:url" : "http://www.laweekly.com/arts/rock-with-me-2374144",
"article:published_time" : "2005-11-14T20:11:46-08:00",
"article:modified_time" : "2014-11-25T18:43:01-08:00",
"article:section" : "Arts",
"article:tag" : "Malibu Creek State",
"og:image" : "http://images1.laweekly.com/imager/rock-with-me/u/original/2444410/shimmy1.jpg",
"og:image:height" : "666",
"og:image:width" : "500",
"og:site_name" : "LA Weekly",
"twitter:title" : "Rock With Me",
"twitter:description" : ""How many boulders can we possibly have to climb over?" I chuckled to my friend Julie, when she warned me that her boyfriend David's friend Ben had aske...",
"twitter:card" : "summary",
"twitter:image" : "http://images1.laweekly.com/imager/rock-with-me/u/original/2444410/shimmy1.jpg",
"twitter:site" : "@laweekly"
} ],
"machine" : {
"os" : "win 7",
"ram" : 13958643712
},
"@version" : "1"
}
} ]
}
#############################################################################################
虚拟机logstash
[root@logstash ~]# cd /etc/logstash/
[root@logstash logstash]# ls
conf.d
[root@logstash logstash]# vim logstash.conf
input{
stdin{}
}
filter{}
output{
stdout{}
}
[root@logstash logstash]# cd /opt/logstash/bin/
[root@logstash bin]# ls
logstash logstash.bat logstash.lib.sh logstash-plugin logstash-plugin.bat plugin plugin.bat rspec rspec.bat setup.bat
[root@logstash bin]# alias logstash=/opt/logstash/bin/logstash
[root@logstash bin]# cd /etc/logstash/
[root@logstash logstash]# ls
conf.d logstash.conf
[root@logstash logstash]# logstash -f logstash.conf 用-f来指定文件路径
Settings: Default pipeline workers: 2
Pipeline main started
aaaaaaaaaaaaaaaaaaaaa
2018-12-26T06:26:10.830Z logstash aaaaaaaaaaaaaaaaaaaaa
bbbbbbbbbbbb
2018-12-26T06:26:13.853Z logstash bbbbbbbbbbbb
cccccccccccc
2018-12-26T06:26:16.189Z logstash cccccccccccc
ssssssssssss
2018-12-26T06:26:18.741Z logstash ssssssssssss
ddddddddddddddddd
2018-12-26T06:26:21.570Z logstash ddddddddddddddddd
kkkkkkk
2018-12-26T06:26:24.609Z logstash kkkkkkk
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Received shutdown signal, but pipeline is still waiting for in-flight events
to be processed. Sending another ^C will force quit Logstash, but this may cause
data loss. {:level=>:warn}
^CSIGINT received. Terminating immediately.. {:level=>:fatal}
########################################################################################
参考：
[root@logstash logstash]# logstash --help
Usage:
/bin/logstash agent [OPTIONS]
Options:
-f, --config CONFIG_PATH Load the logstash config from a specific file
or directory. If a directory is given, all
files in that directory will be concatenated
in lexicographical order and then parsed as a
single config file. You can also specify
wildcards (globs) and any matched files will
be loaded in the order described above.
-e CONFIG_STRING Use the given string as the configuration
data. Same syntax as the config file. If no
input is specified, then the following is
used as the default input:
"input { stdin { type => stdin } }"
and if no output is specified, then the
following is used as the default output:
"output { stdout { codec => rubydebug } }"
If you wish to use both defaults, please use
the empty string for the '-e' flag.
(default: "")
-w, --pipeline-workers COUNT Sets the number of pipeline workers to run.
(default: 2)
-b, --pipeline-batch-size SIZE Size of batches the pipeline is to work in.
(default: 125)
-u, --pipeline-batch-delay DELAY_IN_MS When creating pipeline batches, how long to wait while polling
for the next event.
(default: 5)
--filterworkers COUNT DEPRECATED. Now an alias for --pipeline-workers and -w
-l, --log FILE Write logstash internal logs to the given
file. Without this flag, logstash will emit
logs to standard output.
-v Increase verbosity of logstash internal logs.
Specifying once will show 'informational'
logs. Specifying twice will show 'debug'
logs. This flag is deprecated. You should use
--verbose or --debug instead.
--quiet Quieter logstash logging. This causes only
errors to be emitted.
--verbose More verbose logging. This causes 'info'
level logs to be emitted.
--debug Most verbose logging. This causes 'debug'
level logs to be emitted.
--debug-config Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
in plaintext passwords appearing in your logs!
(default: false)
-V, --version Emit the version of logstash and its friends,
then exit.
-p, --pluginpath PATH A path of where to find plugins. This flag
can be given multiple times to include
multiple paths. Plugins are expected to be
in a specific directory hierarchy:
'PATH/logstash/TYPE/NAME.rb' where TYPE is
'inputs' 'filters', 'outputs' or 'codecs'
and NAME is the name of the plugin.
-t, --configtest Check configuration for valid syntax and then exit.
--[no-]allow-unsafe-shutdown Force logstash to exit during shutdown even
if there are still inflight events in memory.
By default, logstash will refuse to quit until all
received events have been pushed to the outputs.
(default: false)
-r, --[no-]auto-reload Monitor configuration changes and reload
whenever it is changed.
NOTE: use SIGHUP to manually reload the config
(default: false)
--reload-interval RELOAD_INTERVAL How frequently to poll the configuration location
for changes, in seconds.
(default: 3)
--allow-env EXPERIMENTAL. Enables templating of environment variable
values. Instances of "${VAR}" in strings will be replaced
with the respective environment variable value named "VAR".
(default: false)
-h, --help print help
########################################################################################
参考：
[root@logstash bin]# cd /opt/logstash/bin/
[root@logstash bin]# ls
logstash logstash.bat logstash.lib.sh logstash-plugin logstash-plugin.bat plugin plugin.bat rspec rspec.bat setup.bat
[root@logstash bin]# ./logstash-plugin list
Ignoring ffi-1.9.13 because its extensions are not built. Try: gem pristine ffi --version 1.9.13
logstash-codec-collectd
logstash-codec-dots
logstash-codec-edn
logstash-codec-edn_lines
logstash-codec-es_bulk
logstash-codec-fluent
logstash-codec-graphite
logstash-codec-json
logstash-codec-json_lines
logstash-codec-line
logstash-codec-msgpack
logstash-codec-multiline
logstash-codec-netflow
logstash-codec-oldlogstashjson
logstash-codec-plain
logstash-codec-rubydebug
logstash-filter-anonymize
logstash-filter-checksum
logstash-filter-clone
logstash-filter-csv
logstash-filter-date
logstash-filter-dns
logstash-filter-drop
logstash-filter-fingerprint
logstash-filter-geoip
logstash-filter-grok
logstash-filter-json
logstash-filter-kv
logstash-filter-metrics
logstash-filter-multiline
logstash-filter-mutate
logstash-filter-ruby
logstash-filter-sleep
logstash-filter-split
logstash-filter-syslog_pri
logstash-filter-throttle
logstash-filter-urldecode
logstash-filter-useragent
logstash-filter-uuid
logstash-filter-xml
logstash-input-beats
logstash-input-couchdb_changes
logstash-input-elasticsearch
logstash-input-eventlog
logstash-input-exec
logstash-input-file
logstash-input-ganglia
logstash-input-gelf
logstash-input-generator
logstash-input-graphite
logstash-input-heartbeat
logstash-input-http
logstash-input-http_poller
logstash-input-imap
logstash-input-irc
logstash-input-jdbc
logstash-input-kafka
logstash-input-log4j
logstash-input-lumberjack
logstash-input-pipe
logstash-input-rabbitmq
logstash-input-redis
logstash-input-s3
logstash-input-snmptrap
logstash-input-sqs
logstash-input-stdin
logstash-input-syslog
logstash-input-tcp
logstash-input-twitter
logstash-input-udp
logstash-input-unix
logstash-input-xmpp
logstash-input-zeromq
logstash-output-cloudwatch
logstash-output-csv
logstash-output-elasticsearch
logstash-output-email
logstash-output-exec
logstash-output-file
logstash-output-ganglia
logstash-output-gelf
logstash-output-graphite
logstash-output-hipchat
logstash-output-http
logstash-output-irc
logstash-output-juggernaut
logstash-output-kafka
logstash-output-lumberjack
logstash-output-nagios
logstash-output-nagios_nsca
logstash-output-null
logstash-output-opentsdb
logstash-output-pagerduty
logstash-output-pipe
logstash-output-rabbitmq
logstash-output-redis
logstash-output-s3
logstash-output-sns
logstash-output-sqs
logstash-output-statsd
logstash-output-stdout
logstash-output-tcp
logstash-output-udp
logstash-output-xmpp
logstash-output-zeromq
logstash-patterns-core
#############################################################################################
网页访问插件文档
https://www.elastic.co/guide/en/logstash/current/index.html
在Input plugins 那里点击左边的+，右键file，选择在新标签中打开
#############################################################################################
虚拟机logstash
[root@logstash logstash]# cd /etc/logstash
[root@logstash logstash]# vim logstash.conf
input{
stdin{ codec => "json" }
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
aaaaaaaaaaa
{
"message" => "aaaaaaaaaaa",
"tags" => [
[0] "_jsonparsefailure"
],
"@version" => "1",
"@timestamp" => "2018-12-26T06:49:08.292Z",
"host" => "logstash"
}
{"a":1,"b":2,"c":3}
{
"a" => 1,
"b" => 2,
"c" => 3,
"@version" => "1",
"@timestamp" => "2018-12-26T06:49:36.806Z",
"host" => "logstash"
}
#############################################################################################
虚拟机logstash
[root@logstash logstash]# vim logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# touch /tmp/a.log
[root@logstash logstash]# touch /var/tmp/b.log
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
另开一个终端，给2个文件输入东西
[root@logstash ~]# echo A_${RANDOM}
A_19459
[root@logstash ~]# echo A_${RANDOM} >> /tmp/a.log
[root@logstash ~]# echo B_${RANDOM} >> /var/tmp/b.log
在原来的终端会发现跳出来，我们在另一个终端里，往文件里写入的信息
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
会跳出来如下信息：
{
"message" => "A_26708",
"@version" => "1",
"@timestamp" => "2018-12-26T07:31:15.675Z",
"path" => "/tmp/a.log",
"host" => "logstash"
}
{
"message" => "B_21",
"@version" => "1",
"@timestamp" => "2018-12-26T07:31:30.707Z",
"path" => "/var/tmp/b.log",
"host" => "logstash"
}
两次ctrl+c退出
然后再往文件写入内容
[root@logstash logstash]# echo aaa >>/tmp/a.log
[root@logstash logstash]# echo bbb>>/var/tmp/b.log
再次启动服务，还能读出上次没有读完的新内容
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "aaa",
"@version" => "1",
"@timestamp" => "2018-12-26T07:40:19.504Z",
"path" => "/tmp/a.log",
"host" => "logstash"
}
{
"message" => "bbb",
"@version" => "1",
"@timestamp" => "2018-12-26T07:40:19.605Z",
"path" => "/var/tmp/b.log",
"host" => "logstash"
}
[root@logstash logstash]# cat /etc/logstash/logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb" 增加这行，指定新的读取文件的位置
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# ls -A /root/
.sincedb_e9a1772295a869da80134b5c4e75816e ... ... 这是默认的指定读取文件位置的文件
[root@logstash logstash]# cat /root/.sincedb_e9a1772295a869da80134b5c4e75816e
467997 0 64769 12
3984678 0 64769 5
[root@logstash logstash]# rm -rf /root/.sincedb_e9a1772295a869da80134b5c4e75816e
[root@logstash ~]# echo 11111 >> /tmp/a.log
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
无输出
另开终端
[root@logstash ~]# echo 22222 >> /var/tmp/b.log
原终端有输出了
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "22222",
"@version" => "1",
"@timestamp" => "2018-12-26T08:02:48.169Z",
"path" => "/var/tmp/b.log",
"host" => "logstash"
}
[root@logstash logstash]# ls /var/lib/logstash/ 此时已经生成了指定读取文件位置的文件，里面有偏移量
sincedb
[root@logstash logstash]# cat /etc/logstash/logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning" 指定从开始的偏移量读
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
这里会从头开始显示2个文件里的所有内容。
[root@logstash logstash]# cat /etc/logstash/logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning" 指定从开始的偏移量读
type => "test log" 指定类型
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# echo test >> /tmp/a.log 写入新内容
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "test",
"@version" => "1",
"@timestamp" => "2018-12-26T08:07:12.887Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "test log" 这里有显示类型
}
##############################################################################################
[root@logstash logstash]# cat logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning"
type => "test log"
}
tcp {
mode => "server"
host=> "0.0.0.0"
port => 8888
type => "tcp log"
}
udp {
port => 8888
type => "udp log"
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
新开一个终端
[root@logstash fd]# ss -tunlp | grep 8888
udp UNCONN 0 0 :::8888 :::* users:(("java",pid=12416,fd=37))
tcp LISTEN 0 50 :::8888 :::* users:(("java",pid=12416,fd=7))
#####################################################################################################
虚拟机kibana
[root@kibana ~]# echo 0000000 >/dev/udp/192.168.3.200/8888
######################################################################################################
虚拟机logstash
会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "0000000\n", 这是客户端输入的信息
"@version" => "1",
"@timestamp" => "2018-12-26T09:06:52.222Z",
"type" => "udp log", 这是udp协议连接的
"host" => "192.168.3.116" 这是客户端的IP
}
#####################################################################################################
虚拟机kibana
[root@kibana ~]# echo 9999999 >/dev/tcp/192.168.3.200/8888
######################################################################################################
虚拟机logstash
会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
... ...
{
"message" => "9999999",
"@version" => "1",
"@timestamp" => "2018-12-26T09:08:53.830Z",
"host" => "192.168.3.116",
"port" => 54206,
"type" => "tcp log"
}
######################################################################################################
虚拟机logstash
[root@logstash fd]# cd /proc/self/fd
[root@logstash fd]# ls
0 1 2 255
[root@logstash fd]# ll
总用量 0
lrwx------ 1 root root 64 12月 26 15:30 0 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 1 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 2 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 16:45 255 -> /dev/pts/1
[root@logstash fd]# echo haha >./1
haha
[root@logstash fd]# exec 8<>/dev/tcp/192.168.3.200/8888
[root@logstash fd]# ll
总用量 0
lrwx------ 1 root root 64 12月 26 15:30 0 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 1 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 2 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 16:45 255 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 17:14 8 -> socket:[31877]
[root@logstash fd]# echo 88888888888 >&8
[root@logstash fd]# ls
0 1 2 255 8
######################################################################################################
虚拟机logstash
会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
... ...
{
"message" => "88888888888",
"@version" => "1",
"@timestamp" => "2018-12-26T09:15:14.316Z",
"host" => "192.168.3.200",
"port" => 59430,
"type" => "tcp log"
}
######################################################################################################
虚拟机logstash
[root@logstash fd]# ll
总用量 0
... ...
lrwx------ 1 root root 64 12月 26 17:14 8 -> socket:[31877] 这里有8
[root@logstash fd]# ls
0 1 2 255 8 这里有8
[root@logstash fd]# exec 8<&- 删去8
[root@logstash fd]# ll
已经没有8
[root@logstash fd]# ls
0 1 2 255 已经没有8了
.每一个进程都有自己的文件描述符集合.
.当创建进程时,通常有3个打开文件描述符(0,1,2),0代表标准输入,1代表标准输出,2代表标准错误,它们统称为标准IO.
子目录/proc/self本身就是当前运行进程ID的符号链接.
用ls -ld查看/proc/self目录的符号链接,发现每次都不一样,说明我们每次用ls命令时的进程ID都是不同的.
ls -ld /proc/self
lrwxrwxrwx 1 root root 64 2010-10-10 06:25 /proc/self -> 30525
我们查看/proc/self/fd目录下的文件描述符,如下:
ls -l /proc/self/fd
total 0
lrwx------ 1 root root 64 2010-10-10 12:16 0 -> /dev/pts/1
lrwx------ 1 root root 64 2010-10-10 12:16 1 -> /dev/pts/1
lrwx------ 1 root root 64 2010-10-10 12:16 2 -> /dev/pts/1
lr-x------ 1 root root 64 2010-10-10 12:16 3 -> /proc/30578/fd
我们看到了3个标准的IO描述符,它们都被软锭接到了/dev/pts/1,/dev/pts/1是我们通过ssh打开第2个终端,如果是第1个终端,那将是/dev/pts/0.
如果我们通过ipmi的串口登录,这里应该是/dev/ttySx,而如果是本地登录那应该是/dev/ttyx,如果是单用户登录那将是/dev/console.
/dev/pts/x是虚拟终端
/dev/ttySx是串行控制端
/dev/ttyx是控制台
/dev/console是单用户控制台
#####################################################################################################
虚拟机kibana
定义一个函数send()
[root@kibana ~]# function send() {
> exec 9<>/dev/tcp/192.168.3.200/8888
> echo #1 >&9
> exec 9<&-
> }
[root@kibana ~]# send hello
######################################################################################################
虚拟机logstash
之前一直开着的程序，会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出以下信息：
{
"message" => "\e]0;root@kibana:~\a\e]0;root@kibana:~\a",
"@version" => "1",
"@timestamp" => "2018-12-26T11:00:14.992Z",
"host" => "192.168.3.116",
"port" => 46972,
"type" => "tcp log"
}
######################################################################################################
虚拟机logstash
[root@logstash 11729]# man logger
在最后面的几行里找到
EXAMPLES
logger System rebooted
logger -p local0.notice -t HOSTIDM -f /dev/idmc 复制这行
logger -n loghost.example.com System rebooted
粘贴到命令行，然后修改
[root@logstash 11729]# logger -p local0.info -t testlog aaaaaaaaaaaa
等级 标题 内容
[root@logstash 11729]# tail -1 /var/log/messages
Dec 26 17:49:30 logstash testlog: aaaaaaaaaaaa
[root@logstash 11729]# vim /etc/rsyslog.conf
74 local0.info /var/log/info.log 增加这行
[root@logstash 11729]# systemctl restart rsyslog
[root@logstash 11729]# logger -p local0.info -t testlog bbbbb
[root@logstash 11729]# tail -1 /var/log/info.log
Dec 26 17:56:41 logstash testlog: bbbbb
######################################################################################################
虚拟机kibana
[root@kibana ~]# vim /etc/rsyslog.conf
74 local0.info @@192.168.3.200:514 增加74行
90 #*.* @@remote-host:514 第90行有注释说明，把信息写入远程的哪个主机的哪个端口
[root@kibana ~]# systemctl restart rsyslog
######################################################################################################
虚拟机logstash
[root@logstash logstash]# vim logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning"
type => "test log"
}
tcp {
mode => "server"
host=> "0.0.0.0"
port => 8888
type => "tcp log"
}
udp {
port => 8888
type => "udp log"
}
syslog {
port => 514
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
打开程序，一直开着
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
######################################################################################################
新开终端虚拟机logstash
可以查看到已经启动了端口514
[root@logstash 11729]# ss -tunlp | grep 514
udp UNCONN 0 0 :::514 :::* users:(("java",pid=12577,fd=28))
tcp LISTEN 0 50 :::514 :::* users:(("java",pid=12577,fd=40))
######################################################################################################
虚拟机kibana
[root@kibana ~]# logger -p local0.info -t testlog kkkkk
######################################################################################################
虚拟机logstash
之前一直开着的程序，会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出以下信息：
{
"message" => "kkkkk\n", 内容
"@version" => "1",
"@timestamp" => "2018-12-26T10:09:43.000Z",
"host" => "192.168.3.116", 哪个IP的主机写的
"priority" => 134,
"timestamp" => "Dec 26 18:09:43",
"logsource" => "kibana", 主机kibana的日志
"program" => "testlog",
"severity" => 6,
"facility" => 16,
"facility_label" => "local0",
"severity_label" => "Informational" 等级
}
######################################################################################################
虚拟机kibana
工作中的用途，如把有关登陆的日志信息发送给远程的某个主机
[root@logstash logstash]# vim logstash.conf
58 authpriv.* @@192.168.3.200:514 增加58行
[root@kibana ~]# systemctl restart rsyslog
[root@kibana ~]# exit
登出
Connection to 192.168.3.116 closed.
[root@room9pc01 qemu]#
######################################################################################################
虚拟机logstash
之前一直开着的程序，会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出以下信息：
{
"message" => "Unregistered Authentication Agent for unix-process:1289:8507839 (system bus name :1.65, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale zh_CN.UTF-8) (disconnected from bus)\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:24.000Z",
"host" => "192.168.3.116",
"priority" => 85,
"timestamp" => "Dec 26 18:15:24",
"logsource" => "kibana",
"program" => "polkitd",
"pid" => "476",
"severity" => 5,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Notice"
}
{
"message" => "Received disconnect from 192.168.3.254 port 34262:11: disconnected by user\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:37.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:15:37",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1061",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "Disconnected from 192.168.3.254 port 34262\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:37.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:15:37",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1061",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "pam_unix(sshd:session): session closed for user root\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:37.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:15:37",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1061",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
#####################################################################################
真机连接kibana，故意输错密码
[root@room9pc01 qemu]# ssh 192.168.3.116
root@192.168.3.116's password:
Permission denied, please try again.
root@192.168.3.116's password:
######################################################################################################
虚拟机logstash
之前一直开着的程序，会跳出新的信息,
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出以下信息：
{
"message" => "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.168.3.254 user=root\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:17:02.000Z",
"host" => "192.168.3.116",
"priority" => 85,
"timestamp" => "Dec 26 18:17:02",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1302",
"severity" => 5,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Notice"
}
{
"message" => "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:17:02.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:17:02",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1302",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "Failed password for root from 192.168.3.254 port 35864 ssh2\n", 从主机3.254远程连接kibana失败
"@version" => "1",
"@timestamp" => "2018-12-26T10:17:04.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:17:04",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1302",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
#####################################################################################
真机成功连接kibana
[root@room9pc01 qemu]# ssh 192.168.3.116
root@192.168.3.116's password:
Last failed login: Wed Dec 26 18:17:04 CST 2018 from 192.168.3.254 on ssh:notty
There was 1 failed login attempt since the last successful login.
Last login: Wed Dec 26 09:01:00 2018 from 192.168.3.254
[root@kibana ~]#
######################################################################################################
虚拟机logstash
之前一直开着的程序，会跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出以下信息：
{
"message" => "Accepted password for root from 192.168.3.254 port 35868 ssh2\n", 从主机3.254成功远程连接kibana
"@version" => "1",
"@timestamp" => "2018-12-26T10:19:08.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:19:08",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1304",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "pam_unix(sshd:session): session opened for user root by (uid=0)\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:19:08.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:19:08",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1304",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
###########################################################################################
虚拟机logstash
[root@logstash ~]# curl -v www.baidu.com
显示如下：
* About to connect() to www.baidu.com port 80 (#0)
* Trying 14.215.177.39...
* Connected to www.baidu.com (14.215.177.39) port 80 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: www.baidu.com
> Accept: */*
>
< HTTP/1.1 200 OK
< Accept-Ranges: bytes
< Cache-Control: private, no-cache, no-store, proxy-revalidate, no-transform
< Connection: Keep-Alive
< Content-Length: 2381
< Content-Type: text/html
< Date: Wed, 26 Dec 2018 11:46:32 GMT
< Etag: "588604dd-94d"
< Last-Modified: Mon, 23 Jan 2017 13:27:57 GMT
< Pragma: no-cache
< Server: bfe/1.0.8.18
< Set-Cookie: BDORZ=27315; max-age=86400; domain=.baidu.com; path=/
<
<!DOCTYPE html>
... ...
[root@logstash ~]# vim baidu.sh
#!/bin/bash
exec 5<>/dev/tcp/www.baidu.com/80
echo -ne "GET / HTTP/1.1\r\n" >&5
echo -ne "User-Agent: curl/7.29.0\r\n" >&5
echo -ne "Host: www.baidu.com\r\n" >&5
echo -ne "\r\n" >&5
cat <&5
exec 5<&-
[root@logstash ~]# chmod +x baidu.sh
[root@logstash ~]# ./baidu.sh
显示如下：
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: private, no-cache, no-store, proxy-revalidate, no-transform
Connection: Keep-Alive
Content-Length: 2381
Content-Type: text/html
Date: Wed, 26 Dec 2018 11:51:50 GMT
Etag: "588604dd-94d"
Last-Modified: Mon, 23 Jan 2017 13:27:57 GMT
Pragma: no-cache
Server: bfe/1.0.8.18
Set-Cookie: BDORZ=27315; max-age=86400; domain=.baidu.com; path=/
<!DOCTYPE html>
[root@logstash fd]# cd /proc/self/fd
[root@logstash fd]# exec 6<>/etc/passwd
[root@logstash fd]# ll
总用量 0
... ...
lrwx------ 1 root root 64 12月 26 19:11 6 -> /etc/passwd 这里的6是/etc/passwd的快捷方式
[root@logstash fd]# cat /etc/passwd
[root@logstash fd]# cat /etc/passwd | wc -l
20
[root@logstash fd]# cat ./6 查看当前目录下的6，就相当于查看/etc/passwd
[root@logstash fd]# cat ./6 | wc -l
20
######################################################################################################
虚拟机web
[root@web httpd]# cat /var/log/httpd/access_log
192.168.3.254 - - [27/Dec/2018:10:51:23 +0800] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
[root@web httpd]# vim /etc/httpd/conf/httpd.conf
196 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
197 LogFormat "%h %l %u %t \"%r\" %>s %b" common
198
199 <IfModule logio_module>
200 # You need to enable mod_logio.c to use %I and %O
201 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
202 </IfModule>
... ...
217 CustomLog "logs/access_log" combined
apache官方文档
http://httpd.apache.org/
中文版的翻译可以百度一下：
金步国
http://www.jinbuguo.com/
http://www.jinbuguo.com/apache/menu22/mod/mod_log_config.html#logformat
[root@logstash ~]# cd /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/
[root@logstash patterns]# ls
aws bro firewalls haproxy junos mcollective mongodb postgresql redis
bacula exim grok-patterns java linux-syslog mcollective-patterns nagios rails ruby
[root@logstash patterns]# vim grok-patterns
这是创作者提前写好的正则表达式变量，我们可以调用
[root@logstash patterns]# cat grok-patterns
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
HTTPDUSER %{EMAILADDRESS}|%{USER}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
IPORHOST (?:%{IP}|%{HOSTNAME})
HOSTPORT %{IPORHOST}:%{POSINT}
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>\d\d){1,2}
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG [\x21-\x5a\x5c\x5e-\x7e]+
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
# Log Levels
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
######################################################################################################
虚拟机web
[root@room9pc01 qemu]# scp '/root/桌面/filebeat-1.2.3-x86_64.rpm' 192.168.3.100:/root/
[root@web ~]# rpm -ivh filebeat-1.2.3-x86_64.rpm
[root@web ~]# vim /etc/filebeat/filebeat.yml
15 - /var/log/httpd/access_log 改
72 document_type: apachelog 打开注释并修改
278 logstash: 打开注释
279 # The Logstash hosts
280 hosts: ["192.168.3.200:5044"] 打开注释并修改
183 # elasticsearch: 注释
188 # hosts: ["localhost:9200"] 注释
[root@web ~]# grep -Pv "^\s*(#|$)" /etc/filebeat/filebeat.yml
filebeat:
prospectors:
-
paths:
- /var/log/httpd/access_log
input_type: log
document_type: apachelog
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["192.168.3.200:5044"]
shipper:
logging:
files:
rotateeverybytes: 10485760 # = 10MB
[root@web ~]# systemctl restart filebeat
把自己的主页设为百度
[root@web ~]# curl www.baidu.com -o /var/www/html/index.html
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 2381 100 2381 0 0 104k 0 --:--:-- --:--:-- --:--:-- 110k
######################################################################################################
虚拟机
[root@logstash ~]# cd /etc/logstash/
[root@logstash logstash]# vim logstash.conf
input{
file {
path => ["/tmp/a.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning"
type => "test log"
}
tcp {
mode => "server"
host=> "0.0.0.0"
port => 8888
type => "tcp log"
}
udp {
port => 8888
type => "udp log"
}
syslog {
port => 514
}
beats {
port => 5044
}
}
filter{
if [type] == "apachelog" {
grok {
match => { message => "%{COMBINEDAPACHELOG}" }
}}
}
output{
stdout{ codec => "rubydebug" }
if [type] == "apachelog" {
elasticsearch {
hosts => ["es1:9200", "es2:9200", "es3:9200"]
index => "weblog-%{+YYYY.MM.dd}"
}}
}
复制web的apache访问日志粘贴到/tmp/a.log
[root@logstash patterns]# cat /tmp/a.log
192.168.3.254 - - [27/Dec/2018:10:51:23 +0800] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
[root@logstash patterns]# curl -XDELETE http://192.168.3.111:9200/*
{"acknowledged":true}[root@logstash patterns]#
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.3.254 - - [27/Dec/2018:10:51:23 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2018-12-27T03:26:18.499Z",
"type" => "apachelog",
"count" => 1,
"input_type" => "log",
"fields" => nil,
"beat" => {
"hostname" => "web",
"name" => "web"
},
"source" => "/var/log/httpd/access_log",
"offset" => 0,
"host" => "web",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"clientip" => "192.168.3.254",
"ident" => "-",
"auth" => "-",
"timestamp" => "27/Dec/2018:10:51:23 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "403",
"bytes" => "4897",
"referrer" => "\"-\"",
"agent" => "\"curl/7.29.0\""
}
{
"message" => "192.168.3.254 - - [27/Dec/2018:10:51:30 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2018-12-27T03:26:18.499Z",
"beat" => {
"hostname" => "web",
"name" => "web"
},
"offset" => 91,
"type" => "apachelog",
"fields" => nil,
"source" => "/var/log/httpd/access_log",
"input_type" => "log",
"count" => 1,
"host" => "web",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"clientip" => "192.168.3.254",
"ident" => "-",
"auth" => "-",
"timestamp" => "27/Dec/2018:10:51:30 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "403",
"bytes" => "4897",
"referrer" => "\"-\"",
"agent" => "\"curl/7.29.0\""
}
真机访问
http://192.168.3.115:9200/_plugin/head/
会发现跳出来新的信息
还可以访问很多次
for i in {1..10} ;do curl 192.168.3.100 ;done
######################################################################################################
虚拟机logstash
man帮助查用tcp/udp协议，传内容给远程的某个主机的某个端口,写的格式模版
[root@logstash ~]# man bash
... ...
REDIRECTION
... ...
Bash handles several filenames specially when they are used in redirections, as described in the following table:
/dev/fd/fd
If fd is a valid integer, file descriptor fd is duplicated.
/dev/stdin
File descriptor 0 is duplicated.
/dev/stdout
File descriptor 1 is duplicated.
/dev/stderr
File descriptor 2 is duplicated.
/dev/tcp/host/port 这是用tcp协议远程传内容给远程的某个主机的某个端口,写的格式模版
If host is a valid hostname or Internet address, and port is an integer port number or service name, bash attempts to open a
TCP connection to the corresponding socket.
/dev/udp/host/port 这是用udp协议远程传内容给远程的某个主机的某个端口,写的格式模版
If host is a valid hostname or Internet address, and port is an integer port number or service name, bash attempts to open a
UDP connection to the corresponding socket.
... ...
######################################################################################################
虚拟机logstash
[root@logstash ~]# echo $$
11729
[root@logstash ~]# pstree -p
systemd(1)─┬─agetty(496)
... ...
│ └─sshd(13431)───bash(13433)
... ...
另开一个新终端，进入python界面
[root@logstash ~]# python
Python 2.7.5 (default, Aug 4 2017, 00:39:18)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-16)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>
原终端
[root@logstash ~]# pstree -p
systemd(1)─┬─agetty(496)
... ...
│ └─sshd(13431)───bash(13433)───python(13472) 多了一个,python，它的pid号是13472
... ...
[root@logstash ~]# cd /proc/13472/fd
[root@logstash fd]# ll 现在只有0 1 2这三个快捷方式
总用量 0
lrwx------ 1 root root 64 12月 27 13:14 0 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 1 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 2 -> /dev/pts/2
写入一个新文件
[root@logstash fd]# vim /root/a.txt
aaa
aaaa
aaaaa
另一个新终端,在python界面
输入以下内容
>>> f =open("/root/a.txt")
>>> type(f)
<type 'file'>
>>> f.fileno()
3
原终端
[root@logstash fd]# ll 多了 3 这个快捷方式
总用量 0
lrwx------ 1 root root 64 12月 27 13:14 0 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 1 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 2 -> /dev/pts/2
lr-x------ 1 root root 64 12月 27 13:20 3 -> /root/a.txt 多了3这个快捷方式，它指向的源文件是/root/a.txt
[root@logstash fd]# cat ./3 读当前目录下的3，就相当于读源文件/root/a.txt
aaa
aaaa
aaaaa
[root@logstash fd]# cat /root/a.txt 可以发现和读3这个快捷方式的内容一样
aaa
aaaa
aaaaa
另一个新终端,在python界面
输入以下内容
>>> import os 导入os模块
>>> os.read(3,99999) 读取os模块，会发现内容是一样的
'aaa\naaaa\naaaaa\n'
>>> os.read(3,99999)
''
>>> f.seek(0,0)
>>> abc = os.read(3,99999) 设置abc这个变量的值
>>> print(abc) 读abc这个变量，就相当于读源文件/root/a.txt
aaa
aaaa
aaaaa
