Skip to main content

Posts

Showing posts from January, 2013

Usage of Apache Sqoop and Samples

hduser@my_hadoop_domain:/usr/local/sqoop/conf$ sqoop import --connect jdbc:mysql://mysql_db_url/MY_SCHEMA --table MY_TABLE --username myDbUser --password myDbUserPass --target-dir dest #or hduser@my_hadoop_domain:/usr/local/sqoop/conf$ sqoop ---connect jdbc:mysql://mysql_db_url/MY_SCHEMA --table MY_TABLE --username myDbUser --password myDbUserPass --as-sequencefile hduser@my_hadoop_domain:/usr/local/sqoop/docs$ hdfs -ls Found 5 items drwxr-xr-x   - hduser supergroup          0 2013-01-31 15:41 /user/hduser/MY_TABLE drwxr-xr-x   - hduser supergroup          0 2013-01-24 09:51 /user/hduser/books drwxr-xr-x   - hduser supergroup          0 2013-01-31 15:15 /user/hduser/dest drwxr-xr-x   - hduser supergroup          0 2013-01-24 14:13 /user/hduser/examples drwxr-xr-x   - hduser supergroup          0 2013-01-24 10:48 /user/hduser/res...

Install Apache Sqoop

SQOOP2 (1.99.1) installation wget http://www.eu.apache.org/dist/sqoop/1.99.1/sqoop-1.99.1-bin-hadoop100.tar.gz cd /usr/local/ && sudo tar \-zxvf /home/hduser/sqoop-1.99.1-bin-hadoop100.tar.gz mv sqoop-1.99.1-bin-hadoop100/ sqoop chown -Rf hduser:hadoop sqoop vim /home/hduser/.bashrc export SQOOP_HOME=/usr/local/sqoop if [ -d "$SQOOP_HOME/bin" ] ; then    PATH=$PATH:$SQOOP_HOME/bin fi # INSTALLING DEPENDENCIES hduser@my_hadoop_domain:/usr/local/sqoop$ ./bin/addtowar.sh -hadoop-version 1.1.1 -hadoop-path /usr/local/hadoop/ cd /home/hduser/jars wget http://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.21/mysql-connector-java-5.1.21.jar # You can add any arbitrary java jar file to Sqoop server using script addtowar.sh with -jars parameter hduser@mypc:/usr/local/sqoop/bin$ ./bin/addtowar.sh -jars /home/hduser/jars/mysql-connector-java-5.1.21.jar hduser@my_hadoop_domain:/usr/local/sqoop$ ./bin/sqoop.sh server start hduser@my_hadoop_domain:/usr/loc...

Hadoop Mongo Map Reduce sample

create TestHadoop collection in test schema and insert 2M doc. myserver:PRIMARY> use test myserver:PRIMARY> for(i=0;i<1000000;i++) { db.TestHadoop.insert({"name":"ercan","number":i})} myserver:PRIMARY> for(i=0;i<1000000;i++) { db.TestHadoop.insert({"name":"can","number":i})} cd /home/hduser/mongo_hadoop mkdir -p myprojects/test_hadoop cd myprojects/test_hadoop mkdir -p  src/main/resources cd src/main/resources vim mongo-test_hadoop.xml <?xml version="1.0"?> <configuration>   <property>     <!-- run the job verbosely ? -->     <name>mongo.job.verbose</name>     <value>true</value>   </property>   <property>     <!-- Run the job in the foreground and wait for response, or background it? -->     <name>mongo.job.background</name>     <value>false</value>   </property> ...

mysql query result export to csv via python

#!/usr/bin/env python import MySQLdb import csv HOST = "myshost" USER = "myuser" NAME = "myDB" PASSWD = "mypass" PORT = "myport" def connect_db(host, port, user, password, db):     try:         return MySQLdb.connect(host=host, port=port, user=user, passwd=password, db=db)     except MySQLdb.Error, e:         sys.stderr.write("[ERROR] % d: % s\n" % (e.args[0], e.args[1]))         return False def main():     QUERY="SELECT FIELD1,FIELD2,FIELD3 FROM MY_TABLE"     FILENAME="/home/ercan/dumps/dump.csv"       db = connect_db(HOST,int(PORT),USER,PASSWD,NAME)     dump_writer = csv.writer(open(FILENAME,'w'), delimiter=',',quotechar="'")     cur = db.cursor()     cur.execute(QUERY)     for row in cur.fetchall() :         dump_writer.writerow(row)       print("mysql query result e...

Installation Pig and using with Hadoop

wget http://www.eu.apache.org/dist/pig/pig-0.10.0/pig-0.10.0.tar.gz sudo cp pig-0.10.0.tar.gz /usr/local/ sudo tar -xzvf pig-0.10.0.tar.gz sudo mv pig-0.10.0/ pig/ vim /home/hduser/.bashrc export PIG_HOME=/usr/local/pig if [ -d "$PIG_HOME/bin" ] ; then    PATH=$PATH:$PIG_HOME/bin fi chown -R hduser:hadoop /usr/local/pig/ vim /home/hduser/categories.csv 1 routines 2 services 3 applications vim /home/hduser/products.csv 1 AdminConsole 2 LogService 3 CustomerService hadoop dfs -put /home/hduser/categories.csv examples/categories.csv hadoop dfs -put /home/hduser/categories.csv examples/products.csv hduser@myhost://home/hduser$ pig 13/01/24 14:14:14 WARN pig.Main: Cannot write to log file: /home/hduser/pig_1359029654768.log 2013-01-24 14:14:14,773 [main] INFO  org.apache.pig.Main - Apache Pig version 0.10.0 (r1328203) compiled Apr 19 2012, 22:54:12 2013-01-24 14:14:14,968 [main] INFO  org.apache.pig.backend.hadoop.executioneng...

Installation Hive on Hadoop

wget http://ftp.itu.edu.tr/Mirror/Apache/hive/hive-0.10.0/hive-0.10.0-bin.tar.gz sudo cp hive-0.10.0-bin.tar.gz /usr/local/ cd /usr/local/ sudo tar -xzvf hive-0.10.0-bin.tar.gz sudo mv hive-0.10.0-bin/ hive/ vim /home/hduser/.bashrc export HIVE_HOME=/usr/local/hive if [ -d "$JAVA_HOME/bin" ] ; then    PATH=$PATH:$HIVE_HOME/bin fi mkdir /usr/local/warehouse chown -R hduser:hadoop hive/ chown -R hduser:hadoop warehouse/ cd /usr/local/warehouse hduser@myhost:/usr/local/warehouse$ hive hive> Ctrl+z vim /home/hduser/hive_examples.csv 1 apache 2 httpd 3 warehouse 4 java hadoop dfs -mkdir examples hadoop dfs -ls . Found 3 items drwxr-xr-x   - hduser supergroup          0 2013-01-24 09:51 /user/hduser/books drwxr-xr-x   - hduser supergroup          0 2013-01-24 12:50 /user/hduser/examples drwxr-xr-x   - hduser supergroup          0 2013-01-24 10:48 ...

Install Hadoop Single Node Cluster and Configuration

sudo apt-get install python-software-install #Hadoop requires Java 1.6+. It is built and tested on Oracle (nee Sun) Java, which is the only "supported" JVM. #Go to this page http://www.oracle.com/technetwork/java/javase/downloads/jdk6u38-downloads-1877406.html   downloaded  jdk-6u38-linux-x64.bin chmod a+x jdk-6u38-linux-x64.bin ./jdk-6u38-linux-x64.bin #During installation it will ask you to register - press ENTER. #JDK 6 package is extracted into ./jdk1.6.0_38 directory, for example ./jdk1.6.0_30. mv jdk1.6.0_38 java-6-oracle #if /usr/lib/jvm not exists sudo mkdir /usr/lib/jvm sudo mv java-6-oracle /usr/lib/jvm/ #if ssh not installed sudo apt-get install ssh sudo apt-get install rsync java -version sudo addgroup hadoop sudo adduser --ingroup hadoop hduser #Configuring SSH su - hduser ssh-keygen -t rsa -P "" cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys ssh localhost sudo vim /etc/sysctl.conf # disable ipv6 net.ip...

Save a document with pymongo

import sys from datetime import datetime from pymongo import Connection from pymongo.errors import ConnectionFailure """ Connect to mongodb """ def connect():     try:         c = Connection(host="localhost", port=27017)     except ConnectionFailure, e:         sys.stderr.write("Could not connect to MongoDB: %s" % e)         sys.exit(1)     return c """ main method """ def main():     c = connect()     db = c["test"]     user_doc = {         "username" : "ercan.can",         "firstname" : "ercan",         "surname" : "can"     }     db.users.insert(user_doc, safe=True)     print "Successfully inserted document: %s" % user_doc if __name__ == "__main__":     main()

python pip, virtualenv, django, pymongo installation

$sudo apt-get install python-setuptools python-dev build-essential $sudo easy_install pip $sudo pip install virtualenv $mkdir -p ~/virtualfiles $cd ~/virtualfiles $virtualenv -p python2.7 virtualenv $source ~/virtualfiles/virtualenv/bin/activate (virtualenv)ercan@mypcname:~/virtualfiles$ pip install django (virtualenv)ercan@mypcname:~/virtualfiles$ django-admin.py startproject myproject (virtualenv)ercan@mypcname:~/virtualfiles$ python manage.py startapp myapp (virtualenv)ercan@mypcname:~/virtualfiles$ python manage.py runserver localhost:8000 (virtualenv)ercan@mypcname:~/virtualfiles$ python -c "import django; print(django.get_version())" (virtualenv)ercan@mypcname:~/virtualfiles$ sudo pip install pymongo

my vimrc

vim /home/ercan/.vimrc syntax on set background=dark set ruler                     set more                     set autoread                 set number                   set hidden set noautowrite             set lazyredraw               set showmode set showcmd set nocompatible             set autoindent smartindent   set smarttab                 set tabstop=2                 set shiftwidth=2 set scrolloff=5             set sidescrolloff=5           set history=200 set backspace=indent,eol,start set linebreak set cmdheight=2       ...

MONGO update and delete queries

PRIMARY> var query = {{name: 'ercan'} PRIMARY> db.users.remove(query) PRIMARY> db.users.update({score: 1}, {$set: {name: 'ercan', dob: new Date('Dec 17, 1999'), likes: ['sport'], gender: 'm', score: 9}},{ upsert: true }) PRIMARY> db.users.find()         { "_id" : ObjectId("50eeef9f6745413ec3aab73e"), "score" : 9, "dob" : ISODate("1981-12-16T21:00:00Z"), "gender" : "m", "likes" : [ "sport" ], "name" : "ercan" } PRIMARY> db.users.update({score: 9}, {$set: {name: 'ercan', dob: new Date(1978, 12, 18, 15, 12), likes: ['sky'], gender: 'm', score: 9}}) PRIMARY> db.users.find() { "_id" : ObjectId("50eeef9f6745413ec3aab73e"), "score" : 9, "dob" : ISODate("1979-01-18T11:12:00Z"), "gender" : "m", "likes" : [ "sky...

mongo installation : gpg key error

$ sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10 Executing: gpg --ignore-time-conflict --no-options --no-default-keyring --secret-keyring /tmp/tmp.0OmgpQGnPi --trustdb-name /etc/apt/trustdb.gpg --keyring /etc/apt/trusted.gpg --primary-keyring /etc/apt/trusted.gpg --keyserver keyserver.ubuntu.com --recv 7F0CEB10 gpg: requesting key 7F0CEB10 from hkp server keyserver.ubuntu.com ?: keyserver.ubuntu.com: Connection timed out gpgkeys: HTTP fetch error 7: couldn't connect: Connection timed out gpg: no valid OpenPGP data found. gpg: Total number processed: 0 The hkp protocol runs on port TCP 11371 , firewall allows access to that port on keyserver.ubuntu.com.