<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<avro host (flume master)>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
## on hadoop1-cls (master node)
bash-4.1$ pwd
/opt/cloudera/parcels/CDH-4.2.1-1.cdh4.2.1.p0.5/lib/flume-ng
bash-4.1$ bin/flume-ng agent --conf ./conf/ -f conf/flume.conf -n collector
bash-4.1$ vi conf/flume.conf
## Sources #########################################################
## Accept Avro data In from the Edge Agents
collector.sources = AvroIn
collector.sources.AvroIn.type = avro
collector.sources.AvroIn.bind = 0.0.0.0
collector.sources.AvroIn.port = 4545
collector.sources.AvroIn.channels = mc1 mc2
## Channels ########################################################
## Source writes to 2 channels, one for each sink (Fan Out)
collector.channels = mc1 mc2
# http://flume.apache.org/FlumeUserGuide.html#memory-channel
collector.channels.mc1.type = memory
collector.channels.mc1.capacity = 100000
collector.channels.mc1.transactionCapacity = 1000
collector.channels.mc2.type = memory
collector.channels.mc2.capacity = 100000
collector.channels.mc2.transactionCapacity = 1000
## Sinks ###########################################################
collector.sinks = LocalOut HadoopOut
## Write copy to Local Filesystem (Debugging)
# http://flume.apache.org/FlumeUserGuide.html#file-roll-sink
# collector.sinks.LocalOut.type = file_roll
# collector.sinks.LocalOut.sink.directory = /var/log/flume
# collector.sinks.LocalOut.sink.rollInterval = 0
# collector.sinks.LocalOut.channel = mc1
## Write to HDFS
# http://flume.apache.org/FlumeUserGuide.html#hdfs-sink
collector.sinks.HadoopOut.type = hdfs
collector.sinks.HadoopOut.channel = mc2
collector.sinks.HadoopOut.hdfs.path = /flume/logtest/%{log_type}/%{host}/%y-%m-%d
collector.sinks.HadoopOut.hdfs.fileType = DataStream
collector.sinks.HadoopOut.hdfs.writeFormat = Text
collector.sinks.HadoopOut.hdfs.rollSize = 0
collector.sinks.HadoopOut.hdfs.rollCount = 10000
collector.sinks.HadoopOut.hdfs.rollInterval = 600
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<avro client>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ vi bin/flume-ng
set this parameter JAVA_OPTS="-Xms1024m -Xmx2048m"
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ vi conf/f.conf
## Flume NG Apache Log Collection
## Refer to https://cwiki.apache.org/confluence/display/FLUME/Getting+Started
##
# http://flume.apache.org/FlumeUserGuide.html#exec-source
agent.sources = apache
agent.sources.apache.type = exec
agent.sources.apache.command = tail -F /home/ercan/Desktop/HADOOP/flume/apache-flume-1.3.1-src/ercan/x.log
agent.sources.apache.batchSize = 10
agent.sources.apache.channels = memoryChannel
agent.sources.apache.interceptors = itime ihost itype
# http://flume.apache.org/FlumeUserGuide.html#timestamp-interceptor
agent.sources.apache.interceptors.itime.type = timestamp
# http://flume.apache.org/FlumeUserGuide.html#host-interceptor
agent.sources.apache.interceptors.ihost.type = host
agent.sources.apache.interceptors.ihost.useIP = false
agent.sources.apache.interceptors.ihost.hostHeader = host
# http://flume.apache.org/FlumeUserGuide.html#static-interceptor
agent.sources.apache.interceptors.itype.type = static
agent.sources.apache.interceptors.itype.key = log_type
agent.sources.apache.interceptors.itype.value = apache_access_combined
# http://flume.apache.org/FlumeUserGuide.html#memory-channel
agent.channels = memoryChannel
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 100000
agent.channels.memoryChannel.transactionCapacity = 1000
## Send to Flume Collector on 10.1.1.145 (Hadoop Master Node)
# http://flume.apache.org/FlumeUserGuide.html#avro-sink
agent.sinks = AvroSink
agent.sinks.AvroSink.type = avro
agent.sinks.AvroSink.channel = memoryChannel
agent.sinks.AvroSink.hostname = 10.1.1.145
agent.sinks.AvroSink.port = 4545
## Debugging Sink, Comment out AvroSink if you use this one
# http://flume.apache.org/FlumeUserGuide.html#file-roll-sink
#agent.sinks = localout
#agent.sinks.localout.type = file_roll
#agent.sinks.localout.sink.directory = /var/log/flume
#agent.sinks.localout.sink.rollInterval = 0
#agent.sinks.localout.channel = memoryChannel
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ for i in {1..100}; do echo "10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 $i" >> /home/ercan/Desktop/HADOOP/flume/apache-flume-1.3.1-src/ercan/x.log ; done
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ tail -f logs/flume.log
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ bin/flume-ng agent -n agent -c conf -f conf/f.conf
[root@hadoop1-cls ~]# hdfs dfs -ls /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/
Found 1 items
-rw-r--r-- 3 hdfs supergroup 8692 2013-06-05 15:01 /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/FlumeData.1370433563656
[root@hadoop1-cls ~]# hdfs dfs -cat /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/FlumeData.1370433563656
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 1
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 2
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 3
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 4
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 5
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 6
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 7
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 8
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 9
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 10
...
# An other configuration for log transfer hdfs.
agent.sources = apache
agent.sources.apache.type = exec
agent.sources.apache.command = tail -F /opt/server/flume-ng/conf/test.log
agent.sources.apache.batchSize = 1
agent.sources.apache.channels = memoryChannel
agent.channels = memoryChannel
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 100
agent.sinks = AvroSink
agent.sinks.AvroSink.type = avro
agent.sinks.AvroSink.channel = memoryChannel
agent.sinks.AvroSink.hostname = hadoop1-cls
agent.sinks.AvroSink.port = 4545
## on hadoop1-cls (master node)
bash-4.1$ pwd
/opt/cloudera/parcels/CDH-4.2.1-1.cdh4.2.1.p0.5/lib/flume-ng
bash-4.1$ bin/flume-ng agent --conf ./conf/ -f conf/flume.conf -n collector
bash-4.1$ vi conf/flume.conf
## Sources #########################################################
## Accept Avro data In from the Edge Agents
collector.sources = AvroIn
collector.sources.AvroIn.type = avro
collector.sources.AvroIn.bind = 0.0.0.0
collector.sources.AvroIn.port = 4545
collector.sources.AvroIn.channels = mc1 mc2
## Channels ########################################################
## Source writes to 2 channels, one for each sink (Fan Out)
collector.channels = mc1 mc2
# http://flume.apache.org/FlumeUserGuide.html#memory-channel
collector.channels.mc1.type = memory
collector.channels.mc1.capacity = 100000
collector.channels.mc1.transactionCapacity = 1000
collector.channels.mc2.type = memory
collector.channels.mc2.capacity = 100000
collector.channels.mc2.transactionCapacity = 1000
## Sinks ###########################################################
collector.sinks = LocalOut HadoopOut
## Write copy to Local Filesystem (Debugging)
# http://flume.apache.org/FlumeUserGuide.html#file-roll-sink
# collector.sinks.LocalOut.type = file_roll
# collector.sinks.LocalOut.sink.directory = /var/log/flume
# collector.sinks.LocalOut.sink.rollInterval = 0
# collector.sinks.LocalOut.channel = mc1
## Write to HDFS
# http://flume.apache.org/FlumeUserGuide.html#hdfs-sink
collector.sinks.HadoopOut.type = hdfs
collector.sinks.HadoopOut.channel = mc2
collector.sinks.HadoopOut.hdfs.path = /flume/logtest/%{log_type}/%{host}/%y-%m-%d
collector.sinks.HadoopOut.hdfs.fileType = DataStream
collector.sinks.HadoopOut.hdfs.writeFormat = Text
collector.sinks.HadoopOut.hdfs.rollSize = 0
collector.sinks.HadoopOut.hdfs.rollCount = 10000
collector.sinks.HadoopOut.hdfs.rollInterval = 600
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<avro client>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ vi bin/flume-ng
set this parameter JAVA_OPTS="-Xms1024m -Xmx2048m"
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ vi conf/f.conf
## Flume NG Apache Log Collection
## Refer to https://cwiki.apache.org/confluence/display/FLUME/Getting+Started
##
# http://flume.apache.org/FlumeUserGuide.html#exec-source
agent.sources = apache
agent.sources.apache.type = exec
agent.sources.apache.command = tail -F /home/ercan/Desktop/HADOOP/flume/apache-flume-1.3.1-src/ercan/x.log
agent.sources.apache.batchSize = 10
agent.sources.apache.channels = memoryChannel
agent.sources.apache.interceptors = itime ihost itype
# http://flume.apache.org/FlumeUserGuide.html#timestamp-interceptor
agent.sources.apache.interceptors.itime.type = timestamp
# http://flume.apache.org/FlumeUserGuide.html#host-interceptor
agent.sources.apache.interceptors.ihost.type = host
agent.sources.apache.interceptors.ihost.useIP = false
agent.sources.apache.interceptors.ihost.hostHeader = host
# http://flume.apache.org/FlumeUserGuide.html#static-interceptor
agent.sources.apache.interceptors.itype.type = static
agent.sources.apache.interceptors.itype.key = log_type
agent.sources.apache.interceptors.itype.value = apache_access_combined
# http://flume.apache.org/FlumeUserGuide.html#memory-channel
agent.channels = memoryChannel
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 100000
agent.channels.memoryChannel.transactionCapacity = 1000
## Send to Flume Collector on 10.1.1.145 (Hadoop Master Node)
# http://flume.apache.org/FlumeUserGuide.html#avro-sink
agent.sinks = AvroSink
agent.sinks.AvroSink.type = avro
agent.sinks.AvroSink.channel = memoryChannel
agent.sinks.AvroSink.hostname = 10.1.1.145
agent.sinks.AvroSink.port = 4545
## Debugging Sink, Comment out AvroSink if you use this one
# http://flume.apache.org/FlumeUserGuide.html#file-roll-sink
#agent.sinks = localout
#agent.sinks.localout.type = file_roll
#agent.sinks.localout.sink.directory = /var/log/flume
#agent.sinks.localout.sink.rollInterval = 0
#agent.sinks.localout.channel = memoryChannel
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ for i in {1..100}; do echo "10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 $i" >> /home/ercan/Desktop/HADOOP/flume/apache-flume-1.3.1-src/ercan/x.log ; done
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ tail -f logs/flume.log
ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ bin/flume-ng agent -n agent -c conf -f conf/f.conf
[root@hadoop1-cls ~]# hdfs dfs -ls /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/
Found 1 items
-rw-r--r-- 3 hdfs supergroup 8692 2013-06-05 15:01 /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/FlumeData.1370433563656
[root@hadoop1-cls ~]# hdfs dfs -cat /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/FlumeData.1370433563656
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 1
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 2
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 3
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 4
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 5
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 6
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 7
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 8
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 9
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 10
...
# An other configuration for log transfer hdfs.
agent.sources = apache
agent.sources.apache.type = exec
agent.sources.apache.command = tail -F /opt/server/flume-ng/conf/test.log
agent.sources.apache.batchSize = 1
agent.sources.apache.channels = memoryChannel
agent.channels = memoryChannel
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 100
agent.sinks = AvroSink
agent.sinks.AvroSink.type = avro
agent.sinks.AvroSink.channel = memoryChannel
agent.sinks.AvroSink.hostname = hadoop1-cls
agent.sinks.AvroSink.port = 4545
Comments
Post a Comment