flume-----发送 sink 2种本地和hdfs

[root@linux05 conf]# vim hdfssink.conf

//从缓存拿出来   直接到hdfs 中  

******************************************

a1.sources=r1
a1.channels=c1
a1.sinks=k1

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/flume
a1.sources.r1.fileSuffix=.fuck

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100

a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://linux04:9000/flumelogs
a1.sinks.k1.hdfs.filePrefix = Syslog
a1.sinks.k1.hdfs.round = true
a1.sinks.k1.hdfs.roundValue = 1
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.writeFormat=Text
a1.sinks.k1.hdfs.rollInterval=0
a1.sinks.k1.hdfs.rollSize=10240
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.idleTimeout=60

a1.sinks.k1.channel=c1
a1.sources.r1.channels=c1

***************************************

[root@linux05 conf]# flume-ng agent -c . -f hdfssink.conf -n a1 -Dflume.root.logger=info

 

[root@linux05 flume]# echo "nihao hdfs sink ">>ee.txt

 

 

 //按照时间创建  

hdfs sink 按照日期创建

[root@linux05 conf]# vim hdfssink1.conf 

a1.sources=r1
a1.channels=c1
a1.sinks=k1

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/flume
a1.sources.r1.fileSuffix=.fuck

a1.sources.r1.fileHeader = false
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = timestamp

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100


a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=hdfs://linuxo4:9000/flumell
a1.sinks.k1.hdfs.rollInterval=60
a1.sinks.k1.hdfs.rollSize=20000
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.idleTimeout=60
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.round=true
a1.sinks.k1.hdfs.roundValue=10
a1.sinks.k1.hdfs.roundUnit=minute
a1.sinks.k1.hdfs.useLocalTimeStamp=true
a1.sinks.k1.hdfs.filePrefix=%Y-%m-%d

a1.sinks.k1.channel=c1
a1.sources.r1.channels=c1
*********************************************************

[root@linux05 conf]# flume-ng agent -c . -f hdfssink1.conf -n a1 -Dflume.root.logger=info,console

[root@linux05 flume]# echo "ccccccc ">>ee.txt

//结果如图

 ***************************************************************************************************************************************************************************************************************

2.将数据存储到本地文件夹

File Roll Sink

vim file.conf

a1.sources=r1
a1.channels=c1
a1.sinks=k1

a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/flume
a1.sources.r1.fileSuffix=.fuck

a1.channels.c1.type=memory
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100


a1.sinks.k1.type = file_roll   //存到本地
a1.sinks.k1.sink.directory = /flume/logs
a1.sinks.k1.sink.serializer = TEXT

a1.sinks.k1.channel=c1
a1.sources.r1.channels=c1
****************************************************************

[root@linux05 conf]# flume-ng agent -c . -f file.conf -n a1 -Dflume.root.logger=info,console

 

 

[root@linux05 flume]# mkdir logs
[root@linux05 flume]# echo "ccccccc ">>e1.txt
[root@linux05 flume]# echo "ccccccc ">>e2.txt
[root@linux05 flume]# echo "ccccccc ">>e3.txt

 

 //因为按照时间生成的  所以滚动数据  第三个有   前两个 都是自动生成的