sudo apt-get purge openjdk*
sudo apt-get install python-software-properties
#install oracle jdk
java -version
sudo addgroup hadoop
sudo adduser --ingroup hadoop hduser
(hadoop-master,hadoop-slave1,hadoop-slave2)
sudo vim /etc/sysctl.conf
# disable ipv6
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
(hadoop-master,hadoop-slave1,hadoop-slave2)
vim /etc/hosts
10.1.1.111 hadoop-master hadoop-master.mydomain.com hadoop-master
10.1.1.112 hadoop-slave1 hadoop-slave1.mydomain.com hadoop-slave1
10.1.1.113 hadoop-slave2 hadoop-slave2.mydomain.com hadoop-slave2
(hadoop-master)
#Configuring SSH
su - hduser
ssh-keygen -t rsa -P ""
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
ssh hadoop-master.mydomain.com
(hadoop-master)
Distribute the SSH public key of hduser@master
hduser@hadoop-master:~$ ssh-copy-id -i $HOME/.ssh/id_rsa.pub hduser@hadoop-slave1
hduser@hadoop-master:~$ ssh-copy-id -i $HOME/.ssh/id_rsa.pub hduser@hadoop-slave2
(hadoop-master,hadoop-slave1,hadoop-slave2)
cd /usr/local
wget http://www.eu.apache.org/dist/hadoop/core/hadoop-1.1.1/hadoop-1.1.1.tar.gz
sudo tar -xzvf hadoop-1.1.1.tar.gz
rm hadoop-1.1.1.tar.gz
sudo mv hadoop-1.1.1 hadoop
sudo chown -R hduser:hadoop /usr/local/hadoop
(hadoop-master,hadoop-slave1,hadoop-slave2)
sudo mkdir -p /app/hadoop/tmp
sudo chown hduser:hadoop /app/hadoop/tmp
sudo chmod 750 /app/hadoop/tmp
root@hadoop-master:/usr/local/hadoop# vim conf/master
hadoop-master
root@hadoop-master:/usr/local/hadoop# vim conf/slaves
hadoop-master
hadoop-slave1
hadoop-slave2
hduser@hadoop-master:~$ ssh hadoop-master
hduser@hadoop-master:~$ ssh hadoop-slave1
hduser@hadoop-master:~$ ssh hadoop-slave2
(hadoop-master,hadoop-slave1,hadoop-slave2)
vim /home/hduser/.bashrc
export HADOOP_HOME=/usr/local/hadoop
if [ -d "$HADOOP_HOME/bin" ] ; then
PATH=$PATH:$HADOOP_HOME/bin
fi
(hadoop-master,hadoop-slave1,hadoop-slave2)
vim conf/hadoop-env.sh
export JAVA_HOME=/usr/lib/jvm/java-6-oracle
export HADOOP_HOME_WARN_SUPPRESS="TRUE"
(hadoop-master,hadoop-slave1,hadoop-slave2)
vim $HADOOP_HOME/conf/core-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/app/hadoop/tmp</value>
<description>A base for other temporary directories.</description>
</property>
<property>
<name>fs.default.name</name>
<value>hdfs://hadoop-master:54310</value>
<description>The name of the default file system. A URI whose
scheme and authority determine the FileSystem implementation. The
uri's scheme determines the config property (fs.SCHEME.impl) naming
the FileSystem implementation class. The uri's authority is used to
determine the host, port, etc. for a filesystem.</description>
</property>
</configuration>
(hadoop-master,hadoop-slave1,hadoop-slave2)
vim $HADOOP_HOME/conf/mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>hadoop-master:54311</value>
<description>The host and port that the MapReduce job tracker runs
at. If "local", then jobs are run in-process as a single map
and reduce task.
</description>
</property>
</configuration>
(hadoop-master,hadoop-slave1,hadoop-slave2)
vim $HADOOP_HOME/conf/hdfs-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>dfs.socket.timeout</name>
<value>360000</value>
<description>Set to 360000 ms (6 mins) - default: 69000 (ms). Increased to allow for loading of extremely large single files.</description>
</property>
<property>
<name>dfs.datanode.socket.write.timeout</name>
<value>3600000</value>
<description>datanode write timeout</description>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
<description>Default block replication.
The actual number of replications can be specified when the file is created.
The default is used if replication is not specified in create time.
</description>
</property>
</configuration>
hduser@hadoop-master:/usr/local/hadoop$ bin/hadoop namenode -format
Warning: $HADOOP_HOME is deprecated.
13/02/01 15:46:03 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG: host = hadoop-master/10.1.1.111
STARTUP_MSG: args = [-format]
STARTUP_MSG: version = 1.1.1
STARTUP_MSG: build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.1 -r 1411108; compiled by 'hortonfo' on Mon Nov 19 10:48:11 UTC 2012
************************************************************/
13/02/01 15:46:03 INFO util.GSet: VM type = 64-bit
13/02/01 15:46:03 INFO util.GSet: 2% max memory = 17.77875 MB
13/02/01 15:46:03 INFO util.GSet: capacity = 2^21 = 2097152 entries
13/02/01 15:46:03 INFO util.GSet: recommended=2097152, actual=2097152
13/02/01 15:46:03 INFO namenode.FSNamesystem: fsOwner=hduser
13/02/01 15:46:04 INFO namenode.FSNamesystem: supergroup=supergroup
13/02/01 15:46:04 INFO namenode.FSNamesystem: isPermissionEnabled=true
13/02/01 15:46:04 INFO namenode.FSNamesystem: dfs.block.invalidate.limit=100
13/02/01 15:46:04 INFO namenode.FSNamesystem: isAccessTokenEnabled=false accessKeyUpdateInterval=0 min(s), accessTokenLifetime=0 min(s)
13/02/01 15:46:04 INFO namenode.NameNode: Caching file names occuring more than 10 times
13/02/01 15:46:04 INFO common.Storage: Image file of size 112 saved in 0 seconds.
13/02/01 15:46:04 INFO namenode.FSEditLog: closing edit log: position=4, editlog=/app/hadoop/tmp/dfs/name/current/edits
13/02/01 15:46:04 INFO namenode.FSEditLog: close success: truncate to 4, editlog=/app/hadoop/tmp/dfs/name/current/edits
13/02/01 15:46:04 INFO common.Storage: Storage directory /app/hadoop/tmp/dfs/name has been successfully formatted.
13/02/01 15:46:04 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at hadoop-master/10.1.1.111
************************************************************/
hduser@hadoop-master:/usr/local/hadoop$ bin/start-dfs.sh
starting namenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-namenode-hadoop-master.out
hadoop-slave1: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-datanode-hadoop-slave1.out
hadoop-master: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-datanode-hadoop-master.out
hadoop-slave2: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-datanode-hadoop-slave2.out
hadoop-master: starting secondarynamenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-secondarynamenode-hadoop-master.out
hduser@hadoop-master:/usr/local/hadoop$
hduser@hadoop-master:/usr/local/hadoop$ /usr/lib/jvm/java-6-oracle/bin/jps
11074 NameNode
11322 DataNode
11658 Jps
11585 SecondaryNameNode
hduser@hadoop-slave1:/usr/local/hadoop$ /usr/lib/jvm/java-6-oracle/bin/jps
6350 Jps
6279 DataNode
hduser@hadoop-master:/usr/local/hadoop$ bin/stop-dfs.sh
stopping namenode
hadoop-master: stopping datanode
hadoop-slave2: stopping datanode
hadoop-slave1: stopping datanode
hadoop-master: stopping secondarynamenode
#MapReduce daemons
hduser@hadoop-master:/usr/local/hadoop$ bin/start-mapred.sh
starting jobtracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-jobtracker-hadoop-master.out
hadoop-slave2: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-tasktracker-hadoop-slave2.out
hadoop-slave1: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-tasktracker-hadoop-slave1.out
hadoop-master: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hduser-tasktracker-hadoop-master.out
hduser@hadoop-master:/usr/local/hadoop$ /usr/lib/jvm/java-6-oracle/bin/jps
13231 Jps
12865 JobTracker
12257 NameNode
12762 SecondaryNameNode
13105 TaskTracker
12500 DataNode
hduser@hadoop-slave1:/usr/local/hadoop/logs$ /usr/lib/jvm/java-6-oracle/bin/jps
6798 DataNode
7116 Jps
7034 TaskTracker
hduser@hadoop-master:/usr/local/hadoop$ hadoop dfs -put /home/hduser/test.txt examples/
hduser@hadoop-master:/usr/local/hadoop$ hadoop dfs -put /home/hduser/aaa.txt examples/
13/02/01 16:41:54 INFO hdfs.DFSClient: Exception in createBlockOutputStream 10.1.1.157:50010 java.io.IOException: Bad connect ack with firstBadLink as 10.1.1.168:50010
13/02/01 16:41:54 INFO hdfs.DFSClient: Abandoning block blk_-7407970097345859457_1003
13/02/01 16:41:54 INFO hdfs.DFSClient: Excluding datanode 10.1.1.168:50010
Comments
Post a Comment