Skip to main content

Hadoop cluster installation and demos


sudo apt-get purge openjdk*
sudo apt-get install python-software-properties

#install oracle jdk

java -version

sudo addgroup hadoop
sudo adduser --ingroup hadoop hduser

(hadoop-master,hadoop-slave1,hadoop-slave2)
sudo vim /etc/sysctl.conf
# disable ipv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1


(hadoop-master,hadoop-slave1,hadoop-slave2)
vim /etc/hosts
10.1.1.111      hadoop-master       hadoop-master.mydomain.com    hadoop-master
10.1.1.112      hadoop-slave1       hadoop-slave1.mydomain.com    hadoop-slave1
10.1.1.113      hadoop-slave2       hadoop-slave2.mydomain.com    hadoop-slave2

(hadoop-master)
#Configuring SSH
su - hduser
ssh-keygen -t rsa -P ""
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
ssh hadoop-master.mydomain.com


(hadoop-master)
Distribute the SSH public key of hduser@master
hduser@hadoop-master:~$ ssh-copy-id -i $HOME/.ssh/id_rsa.pub hduser@hadoop-slave1
hduser@hadoop-master:~$ ssh-copy-id -i $HOME/.ssh/id_rsa.pub hduser@hadoop-slave2


(hadoop-master,hadoop-slave1,hadoop-slave2)
cd /usr/local
wget http://www.eu.apache.org/dist/hadoop/core/hadoop-1.1.1/hadoop-1.1.1.tar.gz
sudo tar -xzvf hadoop-1.1.1.tar.gz
rm hadoop-1.1.1.tar.gz
sudo mv hadoop-1.1.1 hadoop
sudo chown -R hduser:hadoop /usr/local/hadoop


(hadoop-master,hadoop-slave1,hadoop-slave2)
sudo mkdir -p /app/hadoop/tmp
sudo chown hduser:hadoop /app/hadoop/tmp
sudo chmod 750 /app/hadoop/tmp


root@hadoop-master:/usr/local/hadoop# vim conf/master
hadoop-master

root@hadoop-master:/usr/local/hadoop# vim conf/slaves
hadoop-master
hadoop-slave1
hadoop-slave2


hduser@hadoop-master:~$ ssh hadoop-master
hduser@hadoop-master:~$ ssh hadoop-slave1
hduser@hadoop-master:~$ ssh hadoop-slave2


(hadoop-master,hadoop-slave1,hadoop-slave2)
vim /home/hduser/.bashrc
export HADOOP_HOME=/usr/local/hadoop
if [ -d "$HADOOP_HOME/bin" ] ; then
   PATH=$PATH:$HADOOP_HOME/bin
fi

(hadoop-master,hadoop-slave1,hadoop-slave2)
vim conf/hadoop-env.sh
export JAVA_HOME=/usr/lib/jvm/java-6-oracle
export HADOOP_HOME_WARN_SUPPRESS="TRUE"

(hadoop-master,hadoop-slave1,hadoop-slave2)
vim $HADOOP_HOME/conf/core-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
        <property>
          <name>hadoop.tmp.dir</name>
          <value>/app/hadoop/tmp</value>
          <description>A base for other temporary directories.</description>
        </property>

<property>
 <name>fs.default.name</name>
 <value>hdfs://hadoop-master:54310</value>
 <description>The name of the default file system.  A URI whose
 scheme and authority determine the FileSystem implementation.  The
 uri's scheme determines the config property (fs.SCHEME.impl) naming
 the FileSystem implementation class.  The uri's authority is used to
 determine the host, port, etc. for a filesystem.</description>
</property>

</configuration>



(hadoop-master,hadoop-slave1,hadoop-slave2)
vim $HADOOP_HOME/conf/mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
 <name>mapred.job.tracker</name>
 <value>hadoop-master:54311</value>
 <description>The host and port that the MapReduce job tracker runs
 at.  If "local", then jobs are run in-process as a single map
 and reduce task.
 </description>
</property>
</configuration>

(hadoop-master,hadoop-slave1,hadoop-slave2)
vim $HADOOP_HOME/conf/hdfs-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
   <name>dfs.socket.timeout</name>
    <value>360000</value>
     <description>Set to 360000 ms (6 mins) - default: 69000 (ms). Increased to allow for loading of extremely large single files.</description>
</property>
<property>
   <name>dfs.datanode.socket.write.timeout</name>
    <value>3600000</value>
     <description>datanode write timeout</description>
</property>
<property>
 <name>dfs.replication</name>
 <value>3</value>
 <description>Default block replication.
 The actual number of replications can be specified when the file is created.
 The default is used if replication is not specified in create time.
 </description>
</property>
</configuration>



hduser@hadoop-master:/usr/local/hadoop$ bin/hadoop namenode -format
Warning: $HADOOP_HOME is deprecated.

13/02/01 15:46:03 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG:   host = hadoop-master/10.1.1.111
STARTUP_MSG:   args = [-format]
STARTUP_MSG:   version = 1.1.1
STARTUP_MSG:   build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.1 -r 1411108; compiled by 'hortonfo' on Mon Nov 19 10:48:11 UTC 2012
************************************************************/
13/02/01 15:46:03 INFO util.GSet: VM type       = 64-bit
13/02/01 15:46:03 INFO util.GSet: 2% max memory = 17.77875 MB
13/02/01 15:46:03 INFO util.GSet: capacity      = 2^21 = 2097152 entries
13/02/01 15:46:03 INFO util.GSet: recommended=2097152, actual=2097152
13/02/01 15:46:03 INFO namenode.FSNamesystem: fsOwner=hduser
13/02/01 15:46:04 INFO namenode.FSNamesystem: supergroup=supergroup
13/02/01 15:46:04 INFO namenode.FSNamesystem: isPermissionEnabled=true
13/02/01 15:46:04 INFO namenode.FSNamesystem: dfs.block.invalidate.limit=100
13/02/01 15:46:04 INFO namenode.FSNamesystem: isAccessTokenEnabled=false accessKeyUpdateInterval=0 min(s), accessTokenLifetime=0 min(s)
13/02/01 15:46:04 INFO namenode.NameNode: Caching file names occuring more than 10 times
13/02/01 15:46:04 INFO common.Storage: Image file of size 112 saved in 0 seconds.
13/02/01 15:46:04 INFO namenode.FSEditLog: closing edit log: position=4, editlog=/app/hadoop/tmp/dfs/name/current/edits
13/02/01 15:46:04 INFO namenode.FSEditLog: close success: truncate to 4, editlog=/app/hadoop/tmp/dfs/name/current/edits
13/02/01 15:46:04 INFO common.Storage: Storage directory /app/hadoop/tmp/dfs/name has been successfully formatted.
13/02/01 15:46:04 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at hadoop-master/10.1.1.111
************************************************************/





hduser@hadoop-master:/usr/local/hadoop$ bin/start-dfs.sh
starting namenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-namenode-hadoop-master.out
hadoop-slave1: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-datanode-hadoop-slave1.out
hadoop-master: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-datanode-hadoop-master.out
hadoop-slave2: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-datanode-hadoop-slave2.out
hadoop-master: starting secondarynamenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-secondarynamenode-hadoop-master.out




hduser@hadoop-master:/usr/local/hadoop$
hduser@hadoop-master:/usr/local/hadoop$ /usr/lib/jvm/java-6-oracle/bin/jps
11074 NameNode
11322 DataNode
11658 Jps
11585 SecondaryNameNode


hduser@hadoop-slave1:/usr/local/hadoop$ /usr/lib/jvm/java-6-oracle/bin/jps
6350 Jps
6279 DataNode



hduser@hadoop-master:/usr/local/hadoop$ bin/stop-dfs.sh
stopping namenode
hadoop-master: stopping datanode
hadoop-slave2: stopping datanode
hadoop-slave1: stopping datanode
hadoop-master: stopping secondarynamenode


#MapReduce daemons
hduser@hadoop-master:/usr/local/hadoop$ bin/start-mapred.sh
starting jobtracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-jobtracker-hadoop-master.out
hadoop-slave2: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-tasktracker-hadoop-slave2.out
hadoop-slave1: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-tasktracker-hadoop-slave1.out
hadoop-master: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-tasktracker-hadoop-master.out


hduser@hadoop-master:/usr/local/hadoop$ /usr/lib/jvm/java-6-oracle/bin/jps
13231 Jps
12865 JobTracker
12257 NameNode
12762 SecondaryNameNode
13105 TaskTracker
12500 DataNode

hduser@hadoop-slave1:/usr/local/hadoop/logs$ /usr/lib/jvm/java-6-oracle/bin/jps
6798 DataNode
7116 Jps
7034 TaskTracker


hduser@hadoop-master:/usr/local/hadoop$ hadoop dfs -put /home/hduser/test.txt examples/
hduser@hadoop-master:/usr/local/hadoop$ hadoop dfs -put /home/hduser/aaa.txt examples/
13/02/01 16:41:54 INFO hdfs.DFSClient: Exception in createBlockOutputStream 10.1.1.157:50010 java.io.IOException: Bad connect ack with firstBadLink as 10.1.1.168:50010
13/02/01 16:41:54 INFO hdfs.DFSClient: Abandoning block blk_-7407970097345859457_1003
13/02/01 16:41:54 INFO hdfs.DFSClient: Excluding datanode 10.1.1.168:50010

















Comments

Popular posts from this blog

Offline AI Coding in VSCode: Continue + Ollama Complete Setup Guide

Want GitHub Copilot-level code assistance — but fully offline, air-gapped, and private? This guide walks you through setting up Continue (a VSCode extension) with Ollama (a local model runner) so you get chat, autocomplete, and inline editing powered entirely by models running on your own machine. No internet required at runtime. Table of Contents What Are Continue and Ollama? Prerequisites & Hardware Requirements Choosing the Right Models Step 1 — Installing Ollama & Pulling Models Step 2 — Installing Continue in VSCode (Offline) Step 3 — Configuring config.yaml Step 4 — Disabling Telemetry Step 5 — Usage & Shortcuts Troubleshooting What Are Continue and Ollama? Continue is an open-source VSCode (and JetBrains) extension that adds AI-powered coding features directly into your editor — think chat, inline edits, and tab autocomplete. By default it can connect to Claude, GPT-4, or any API. But it also supports local providers , which is exactly what we want here. Ollama i...

Local LLM on Ubuntu (CPU-Only): Ollama + Qwen Coder + Open WebUI + RAG Setup

Local LLM on Ubuntu (CPU-Only): Ollama + Qwen Coder + Open WebUI + RAG Run a private, high-performance internal AI coding assistant on your own hardware—no GPU required. Feb 13, 2026 Qwen 2.5 Coder RAG Enabled Enterprise Ready Keeping your code and documents on your own infrastructure is the gold standard for enterprise security. By combining Ollama , Qwen Coder , and Open WebUI , we can build a production-grade AI assistant that respects your privacy. Environment: CPU-only Ubuntu server (e.g., i9/Threadripper) with 64GB+ RAM for smooth 14B model inference. 1. System Preparation First, update your local environment and install necessary dependencies: sudo apt update && sudo apt upgrade -y sudo apt install -y curl git ca-certificates lscpu 2. Ollama & Model Tuning Install Ollama and optimize the configuration for CPU-only inference by setting the thread count: ...

Defining tomcat and mysql server Amazon EC2 instance

-- Create new amazon ec2 instance http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html -- Give permisson to pem file chmod 400 mycertificate.pem -- Connect to EC2 instance ssh -i mycertificate.pem ubuntu@ec2-51-51-51-51.eu-west-1.compute.amazonaws.com -- Installation Oracle Java 1.8 $ wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz $ tar -xzvf jdk-8u92-linux-x64.tar.gz $ sudo mkdir -p /usr/java/ $ sudo mv jdk1.8.0_92/ /usr/java/ $ sudo vim  /etc/profile         JAVA_HOME=/usr/java/jdk1.8.0_92         PATH=$PATH:$HOME/bin:$JAVA_HOME/bin         JRE_HOME=/usr/java/jdk1.8.0_92/jre         PATH=$PATH:$HOME/bin:$JRE_HOME/bin         export JAVA_HOME...