Skip to main content

Settings Flume Master and Flume client for transfer avro data any client to hdfs

<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<avro host (flume master)>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

## on hadoop1-cls  (master node)

bash-4.1$ pwd
/opt/cloudera/parcels/CDH-4.2.1-1.cdh4.2.1.p0.5/lib/flume-ng
bash-4.1$ bin/flume-ng agent --conf ./conf/ -f conf/flume.conf -n collector


bash-4.1$ vi conf/flume.conf
## Sources #########################################################
## Accept Avro data In from the Edge Agents
collector.sources = AvroIn
collector.sources.AvroIn.type = avro
collector.sources.AvroIn.bind = 0.0.0.0
collector.sources.AvroIn.port = 4545
collector.sources.AvroIn.channels = mc1 mc2

## Channels ########################################################
## Source writes to 2 channels, one for each sink (Fan Out)
collector.channels = mc1 mc2

# http://flume.apache.org/FlumeUserGuide.html#memory-channel
collector.channels.mc1.type = memory
collector.channels.mc1.capacity = 100000
collector.channels.mc1.transactionCapacity = 1000

collector.channels.mc2.type = memory
collector.channels.mc2.capacity = 100000
collector.channels.mc2.transactionCapacity = 1000

## Sinks ###########################################################
collector.sinks = LocalOut HadoopOut

## Write copy to Local Filesystem (Debugging)
# http://flume.apache.org/FlumeUserGuide.html#file-roll-sink
# collector.sinks.LocalOut.type = file_roll
# collector.sinks.LocalOut.sink.directory = /var/log/flume
# collector.sinks.LocalOut.sink.rollInterval = 0
# collector.sinks.LocalOut.channel = mc1

## Write to HDFS
# http://flume.apache.org/FlumeUserGuide.html#hdfs-sink
collector.sinks.HadoopOut.type = hdfs
collector.sinks.HadoopOut.channel = mc2
collector.sinks.HadoopOut.hdfs.path = /flume/logtest/%{log_type}/%{host}/%y-%m-%d
collector.sinks.HadoopOut.hdfs.fileType = DataStream
collector.sinks.HadoopOut.hdfs.writeFormat = Text
collector.sinks.HadoopOut.hdfs.rollSize = 0
collector.sinks.HadoopOut.hdfs.rollCount = 10000
collector.sinks.HadoopOut.hdfs.rollInterval = 600







<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<avro client>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ vi bin/flume-ng
set this parameter           JAVA_OPTS="-Xms1024m -Xmx2048m"



ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ vi conf/f.conf
## Flume NG Apache Log Collection                                                                                                                                                                                  
## Refer to https://cwiki.apache.org/confluence/display/FLUME/Getting+Started
##
# http://flume.apache.org/FlumeUserGuide.html#exec-source
agent.sources = apache
agent.sources.apache.type = exec
agent.sources.apache.command = tail -F /home/ercan/Desktop/HADOOP/flume/apache-flume-1.3.1-src/ercan/x.log
agent.sources.apache.batchSize = 10
agent.sources.apache.channels = memoryChannel
agent.sources.apache.interceptors = itime ihost itype
# http://flume.apache.org/FlumeUserGuide.html#timestamp-interceptor
agent.sources.apache.interceptors.itime.type = timestamp
# http://flume.apache.org/FlumeUserGuide.html#host-interceptor
agent.sources.apache.interceptors.ihost.type = host
agent.sources.apache.interceptors.ihost.useIP = false
agent.sources.apache.interceptors.ihost.hostHeader = host
# http://flume.apache.org/FlumeUserGuide.html#static-interceptor
agent.sources.apache.interceptors.itype.type = static
agent.sources.apache.interceptors.itype.key = log_type
agent.sources.apache.interceptors.itype.value = apache_access_combined

# http://flume.apache.org/FlumeUserGuide.html#memory-channel
agent.channels = memoryChannel
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 100000
agent.channels.memoryChannel.transactionCapacity = 1000



## Send to Flume Collector on 10.1.1.145 (Hadoop Master Node)
# http://flume.apache.org/FlumeUserGuide.html#avro-sink
agent.sinks = AvroSink
agent.sinks.AvroSink.type = avro
agent.sinks.AvroSink.channel = memoryChannel
agent.sinks.AvroSink.hostname = 10.1.1.145
agent.sinks.AvroSink.port = 4545

## Debugging Sink, Comment out AvroSink if you use this one
# http://flume.apache.org/FlumeUserGuide.html#file-roll-sink
#agent.sinks = localout
#agent.sinks.localout.type = file_roll
#agent.sinks.localout.sink.directory = /var/log/flume
#agent.sinks.localout.sink.rollInterval = 0
#agent.sinks.localout.channel = memoryChannel


ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ for i in {1..100}; do  echo "10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 $i" >> /home/ercan/Desktop/HADOOP/flume/apache-flume-1.3.1-src/ercan/x.log ; done

ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ tail -f logs/flume.log

ercan@ercan-can-pc:~/Downloads/apache-flume-1.3.0-bin$ bin/flume-ng agent -n agent -c conf -f conf/f.conf


[root@hadoop1-cls ~]# hdfs dfs -ls /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/
Found 1 items
-rw-r--r--   3 hdfs supergroup       8692 2013-06-05 15:01 /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/FlumeData.1370433563656
[root@hadoop1-cls ~]# hdfs dfs -cat /flume/logtest/apache_access_combined/ercan-can-pc/13-06-05/FlumeData.1370433563656
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 1
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 2
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 3
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 4
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 5
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 6
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 7
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 8
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 9
10.0.0.10 - - [07/May/2013:17:35:35 -0800] 'GET /var/www/view/xxx/yyy HTTP/1.1' 200 10
...





# An other configuration for log transfer hdfs.
agent.sources = apache
agent.sources.apache.type = exec
agent.sources.apache.command = tail -F /opt/server/flume-ng/conf/test.log
agent.sources.apache.batchSize = 1
agent.sources.apache.channels = memoryChannel
agent.channels = memoryChannel
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity = 100
agent.sinks = AvroSink
agent.sinks.AvroSink.type = avro
agent.sinks.AvroSink.channel = memoryChannel
agent.sinks.AvroSink.hostname = hadoop1-cls
agent.sinks.AvroSink.port = 4545



















Comments

Popular posts from this blog

Offline AI Coding in VSCode: Continue + Ollama Complete Setup Guide

Want GitHub Copilot-level code assistance — but fully offline, air-gapped, and private? This guide walks you through setting up Continue (a VSCode extension) with Ollama (a local model runner) so you get chat, autocomplete, and inline editing powered entirely by models running on your own machine. No internet required at runtime. Table of Contents What Are Continue and Ollama? Prerequisites & Hardware Requirements Choosing the Right Models Step 1 — Installing Ollama & Pulling Models Step 2 — Installing Continue in VSCode (Offline) Step 3 — Configuring config.yaml Step 4 — Disabling Telemetry Step 5 — Usage & Shortcuts Troubleshooting What Are Continue and Ollama? Continue is an open-source VSCode (and JetBrains) extension that adds AI-powered coding features directly into your editor — think chat, inline edits, and tab autocomplete. By default it can connect to Claude, GPT-4, or any API. But it also supports local providers , which is exactly what we want here. Ollama i...

Local LLM on Ubuntu (CPU-Only): Ollama + Qwen Coder + Open WebUI + RAG Setup

Local LLM on Ubuntu (CPU-Only): Ollama + Qwen Coder + Open WebUI + RAG Run a private, high-performance internal AI coding assistant on your own hardware—no GPU required. Feb 13, 2026 Qwen 2.5 Coder RAG Enabled Enterprise Ready Keeping your code and documents on your own infrastructure is the gold standard for enterprise security. By combining Ollama , Qwen Coder , and Open WebUI , we can build a production-grade AI assistant that respects your privacy. Environment: CPU-only Ubuntu server (e.g., i9/Threadripper) with 64GB+ RAM for smooth 14B model inference. 1. System Preparation First, update your local environment and install necessary dependencies: sudo apt update && sudo apt upgrade -y sudo apt install -y curl git ca-certificates lscpu 2. Ollama & Model Tuning Install Ollama and optimize the configuration for CPU-only inference by setting the thread count: ...

Defining tomcat and mysql server Amazon EC2 instance

-- Create new amazon ec2 instance http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html -- Give permisson to pem file chmod 400 mycertificate.pem -- Connect to EC2 instance ssh -i mycertificate.pem ubuntu@ec2-51-51-51-51.eu-west-1.compute.amazonaws.com -- Installation Oracle Java 1.8 $ wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz $ tar -xzvf jdk-8u92-linux-x64.tar.gz $ sudo mkdir -p /usr/java/ $ sudo mv jdk1.8.0_92/ /usr/java/ $ sudo vim  /etc/profile         JAVA_HOME=/usr/java/jdk1.8.0_92         PATH=$PATH:$HOME/bin:$JAVA_HOME/bin         JRE_HOME=/usr/java/jdk1.8.0_92/jre         PATH=$PATH:$HOME/bin:$JRE_HOME/bin         export JAVA_HOME...