create TestHadoop collection in test schema and insert 2M doc.
myserver:PRIMARY> use test
myserver:PRIMARY> for(i=0;i<1000000;i++) { db.TestHadoop.insert({"name":"ercan","number":i})}
myserver:PRIMARY> for(i=0;i<1000000;i++) { db.TestHadoop.insert({"name":"can","number":i})}
cd /home/hduser/mongo_hadoop
mkdir -p myprojects/test_hadoop
cd myprojects/test_hadoop
mkdir -p src/main/resources
cd src/main/resources
vim mongo-test_hadoop.xml
<?xml version="1.0"?>
<configuration>
<property>
<!-- run the job verbosely ? -->
<name>mongo.job.verbose</name>
<value>true</value>
</property>
<property>
<!-- Run the job in the foreground and wait for response, or background it? -->
<name>mongo.job.background</name>
<value>false</value>
</property>
<property>
<!-- The field to pass as the mapper key. Defaults to _id if blank -->
<name>mongo.input.key</name>
<value>name</value>
</property>
<property>
<!-- If you are reading from mongo, the URI -->
<name>mongo.input.uri</name>
<value>mongodb://mymongo_server_url/test.TestHadoop</value>
</property>
<property>
<!-- If you are writing to mongo, the URI -->
<name>mongo.output.uri</name>
<value>mongodb://mymongo_server_url/test.TestHadoop_out</value>
</property>
<property>
<!-- The query, in JSON, to execute [OPTIONAL] -->
<name>mongo.input.query</name>
<!--<value>{"x": {"$regex": "^eliot", "$options": ""}}</value>-->
<value></value>
</property>
<property>
<!-- The fields, in JSON, to read [OPTIONAL] -->
<name>mongo.input.fields</name>
<value></value>
</property>
<property>
<!-- A JSON sort specification for read [OPTIONAL] -->
<name>mongo.input.sort</name>
<value></value>
</property>
<property>
<!-- The number of documents to limit to for read [OPTIONAL] -->
<name>mongo.input.limit</name>
<value>0</value> <!-- 0 == no limit -->
</property>
<property>
<!-- The number of documents to skip in read [OPTIONAL] -->
<!-- TODO - Are we running limit() or skip() first? -->
<name>mongo.input.skip</name>
<value>0</value> <!-- 0 == no skip -->
</property>
<property>
<!-- Class for the mapper -->
<name>mongo.job.mapper</name>
<value>com.mongodb.hadoop.testhadoop.TestHadoopMapper</value>
</property>
<property>
<!-- Reducer class -->
<name>mongo.job.reducer</name>
<value>com.mongodb.hadoop.testhadoop.TestHadoopReducer</value>
</property>
<property>
<!-- InputFormat Class -->
<name>mongo.job.input.format</name>
<value>com.mongodb.hadoop.MongoInputFormat</value>
</property>
<property>
<!-- OutputFormat Class -->
<name>mongo.job.output.format</name>
<value>com.mongodb.hadoop.MongoOutputFormat</value>
</property>
<property>
<!-- Output key class for the output format -->
<name>mongo.job.output.key</name>
<value>org.apache.hadoop.io.Text</value>
</property>
<property>
<!-- Output value class for the output format -->
<name>mongo.job.output.value</name>
<value>org.apache.hadoop.io.IntWritable</value>
</property>
<property>
<!-- Output key class for the mapper [optional] -->
<name>mongo.job.mapper.output.key</name>
<value></value>
<value>org.apache.hadoop.io.Text</value>
</property>
<property>
<!-- Output value class for the mapper [optional] -->
<name>mongo.job.mapper.output.value</name>
<value>org.apache.hadoop.io.IntWritable</value>
</property>
<property>
<!-- Class for the combiner [optional] -->
<name>mongo.job.combiner</name>
<value>com.mongodb.hadoop.testhadoop.TestHadoopReducer</value>
</property>
<property>
<!-- Partitioner class [optional] -->
<name>mongo.job.partitioner</name>
<value></value>
</property>
<property>
<!-- Sort Comparator class [optional] -->
<name>mongo.job.sort_comparator</name>
<value></value>
</property>
</configuration>
cd ..
mkdir -p java/com/mongodb/hadoop/testhadoop/
cd java/com/mongodb/hadoop/testhadoop/
vim TestHadoopMapper.java
package com.mongodb.hadoop.testhadoop;
import org.bson.*;
import com.mongodb.hadoop.util.*;
import org.apache.commons.logging.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import java.io.*;
import java.util.*;
/**
* The test hadoop yield mapper.
*/
public class TestHadoopMapper extends Mapper<String, BSONObject, Text, IntWritable> {
@Override
public void map( final String name,final BSONObject doc, final Context pContext ) throws IOException, InterruptedException{
String shape = ( (String) doc.get( "shape" ) );
//int _number = ( (Integer) doc.get( "number" ) );
if(name != null)
pContext.write( new Text( name ), new IntWritable( 1 ) );
//pContext.write( new Text( name ), new IntWritable( _number ) );
}
private static final Log LOG = LogFactory.getLog( TestHadoopMapper.class );
}
vim TestHadoopReducer.java
package com.mongodb.hadoop.testhadoop;
import org.bson.*;
import com.mongodb.hadoop.util.*;
import org.apache.commons.logging.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import java.io.*;
import java.util.*;
/**
* The test hadoop yield reducer.
*/
public class TestHadoopReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
@Override
public void reduce( final Text name, final Iterable<IntWritable> names, final Context pContext ) throws IOException, InterruptedException{
int count = 0;
for ( final IntWritable c: names){
LOG.debug( "name: " + name + " Value: " + c );
count += c.get();
}
pContext.write( name, new IntWritable( count ) );
}
private static final Log LOG = LogFactory.getLog( TestHadoopReducer.class );
}
vim TestHadoopXMLConfig.java
package com.mongodb.hadoop.testhadoop;
import org.bson.*;
import com.mongodb.hadoop.util.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
/**
* The test hadoop yield xml config object.
*/
public class TestHadoopXMLConfig extends MongoTool {
static{
Configuration.addDefaultResource( "mongo-test_hadoop.xml" );
}
public static void main( final String[] pArgs ) throws Exception{
System.exit( ToolRunner.run( new TestHadoopXMLConfig(), pArgs ) );
}
}
cd ~/mongo-hadoop
vim ./project/MongoHadoopBuild.scala
#add the following code line to MongoHadoopBuild.scala file
lazy val testHadoopProject = Project( id = "test-hadoop", base = file("arproject/test_hadoop"), settings = exampleSettings ) dependsOn( core )
cd ~/mongo-hadoop
./sbt test-hadoop/package
hadoop jar myprojects/test_hadoop/target/test-hadoop-1.1.0-SNAPSHOT.jar com.mongodb.hadoop.testhadoop.TestHadoopXMLConfig
Comments
Post a Comment