Skip to main content

Hadoop Mongo Map Reduce sample


create TestHadoop collection in test schema and insert 2M doc.

myserver:PRIMARY> use test
myserver:PRIMARY> for(i=0;i<1000000;i++) { db.TestHadoop.insert({"name":"ercan","number":i})}
myserver:PRIMARY> for(i=0;i<1000000;i++) { db.TestHadoop.insert({"name":"can","number":i})}



cd /home/hduser/mongo_hadoop
mkdir -p myprojects/test_hadoop
cd myprojects/test_hadoop
mkdir -p  src/main/resources
cd src/main/resources


vim mongo-test_hadoop.xml
<?xml version="1.0"?>
<configuration>
  <property>
    <!-- run the job verbosely ? -->
    <name>mongo.job.verbose</name>
    <value>true</value>
  </property>
  <property>
    <!-- Run the job in the foreground and wait for response, or background it? -->
    <name>mongo.job.background</name>
    <value>false</value>
  </property>
  <property>
      <!-- The field to pass as the mapper key. Defaults to _id if blank -->
      <name>mongo.input.key</name>
      <value>name</value>
  </property>
  <property>
    <!-- If you are reading from mongo, the URI -->
    <name>mongo.input.uri</name>
    <value>mongodb://mymongo_server_url/test.TestHadoop</value>
  </property>
  <property>
    <!-- If you are writing to mongo, the URI -->
    <name>mongo.output.uri</name>
    <value>mongodb://mymongo_server_url/test.TestHadoop_out</value>
  </property>
  <property>
    <!-- The query, in JSON, to execute [OPTIONAL] -->
    <name>mongo.input.query</name>
    <!--<value>{"x": {"$regex": "^eliot", "$options": ""}}</value>-->
    <value></value>
  </property>
  <property>
    <!-- The fields, in JSON, to read [OPTIONAL] -->
    <name>mongo.input.fields</name>
    <value></value>
  </property>
  <property>
    <!-- A JSON sort specification for read [OPTIONAL] -->
    <name>mongo.input.sort</name>
    <value></value>
  </property>
  <property>
    <!-- The number of documents to limit to for read [OPTIONAL] -->
    <name>mongo.input.limit</name>
    <value>0</value> <!-- 0 == no limit -->
  </property>
  <property>
    <!-- The number of documents to skip in read [OPTIONAL] -->
    <!-- TODO - Are we running limit() or skip() first? -->
    <name>mongo.input.skip</name>
    <value>0</value> <!-- 0 == no skip -->
  </property>
  <property>
    <!-- Class for the mapper -->
    <name>mongo.job.mapper</name>
    <value>com.mongodb.hadoop.testhadoop.TestHadoopMapper</value>
  </property>
  <property>
    <!-- Reducer class -->
    <name>mongo.job.reducer</name>
    <value>com.mongodb.hadoop.testhadoop.TestHadoopReducer</value>
  </property>
  <property>
    <!-- InputFormat Class -->
    <name>mongo.job.input.format</name>
    <value>com.mongodb.hadoop.MongoInputFormat</value>
  </property>
  <property>
    <!-- OutputFormat Class -->
    <name>mongo.job.output.format</name>
    <value>com.mongodb.hadoop.MongoOutputFormat</value>
  </property>
  <property>
    <!-- Output key class for the output format -->
    <name>mongo.job.output.key</name>
    <value>org.apache.hadoop.io.Text</value>
  </property>
  <property>
    <!-- Output value class for the output format -->
    <name>mongo.job.output.value</name>
    <value>org.apache.hadoop.io.IntWritable</value>
  </property>
  <property>
    <!-- Output key class for the mapper [optional] -->
    <name>mongo.job.mapper.output.key</name>
    <value></value>
<value>org.apache.hadoop.io.Text</value>
  </property>
  <property>
    <!-- Output value class for the mapper [optional] -->
    <name>mongo.job.mapper.output.value</name>
    <value>org.apache.hadoop.io.IntWritable</value>
  </property>
  <property>
    <!-- Class for the combiner [optional] -->
    <name>mongo.job.combiner</name>
    <value>com.mongodb.hadoop.testhadoop.TestHadoopReducer</value>
  </property>
  <property>
    <!-- Partitioner class [optional] -->
    <name>mongo.job.partitioner</name>
    <value></value>
  </property>
  <property>
    <!-- Sort Comparator class [optional] -->
    <name>mongo.job.sort_comparator</name>
    <value></value>
  </property>

</configuration>



cd ..
mkdir -p java/com/mongodb/hadoop/testhadoop/
cd java/com/mongodb/hadoop/testhadoop/





vim TestHadoopMapper.java

package com.mongodb.hadoop.testhadoop;

import org.bson.*;
import com.mongodb.hadoop.util.*;
import org.apache.commons.logging.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import java.io.*;
import java.util.*;

/**
 * The test hadoop yield mapper.
 */
public class TestHadoopMapper extends Mapper<String, BSONObject, Text, IntWritable> {

    @Override
    public void map( final String name,final BSONObject doc, final Context pContext ) throws IOException, InterruptedException{

        String shape = ( (String) doc.get( "shape" ) );
     
        //int _number = ( (Integer) doc.get( "number" ) );

        if(name != null)
        pContext.write( new Text( name ), new IntWritable( 1 ) );
        //pContext.write( new Text( name ), new IntWritable( _number ) );
    }

    private static final Log LOG = LogFactory.getLog( TestHadoopMapper.class );
}




vim TestHadoopReducer.java

package com.mongodb.hadoop.testhadoop;
import org.bson.*;
import com.mongodb.hadoop.util.*;
import org.apache.commons.logging.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import java.io.*;
import java.util.*;

/**
 * The test hadoop yield reducer.
 */
public class TestHadoopReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
    @Override
    public void reduce( final Text name, final Iterable<IntWritable> names, final Context pContext ) throws IOException, InterruptedException{
       
        int count = 0;
       
        for ( final IntWritable c: names){
            LOG.debug( "name: " + name + " Value: " + c );
            count += c.get();
        }

        pContext.write( name, new IntWritable( count ) );
       
    }

    private static final Log LOG = LogFactory.getLog( TestHadoopReducer.class );

}



vim TestHadoopXMLConfig.java

package com.mongodb.hadoop.testhadoop;
import org.bson.*;
import com.mongodb.hadoop.util.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;

/**
 * The test hadoop yield xml config object.
 */
public class TestHadoopXMLConfig extends MongoTool {

    static{
        Configuration.addDefaultResource( "mongo-test_hadoop.xml" );
    }

    public static void main( final String[] pArgs ) throws Exception{
        System.exit( ToolRunner.run( new TestHadoopXMLConfig(), pArgs ) );
    }
}



cd ~/mongo-hadoop
vim ./project/MongoHadoopBuild.scala
#add the following code line to MongoHadoopBuild.scala file
lazy val testHadoopProject  = Project( id = "test-hadoop", base = file("arproject/test_hadoop"),  settings = exampleSettings ) dependsOn( core )

cd ~/mongo-hadoop
./sbt test-hadoop/package

hadoop jar myprojects/test_hadoop/target/test-hadoop-1.1.0-SNAPSHOT.jar com.mongodb.hadoop.testhadoop.TestHadoopXMLConfig




Comments

Popular posts from this blog

Offline AI Coding in VSCode: Continue + Ollama Complete Setup Guide

Want GitHub Copilot-level code assistance — but fully offline, air-gapped, and private? This guide walks you through setting up Continue (a VSCode extension) with Ollama (a local model runner) so you get chat, autocomplete, and inline editing powered entirely by models running on your own machine. No internet required at runtime. Table of Contents What Are Continue and Ollama? Prerequisites & Hardware Requirements Choosing the Right Models Step 1 — Installing Ollama & Pulling Models Step 2 — Installing Continue in VSCode (Offline) Step 3 — Configuring config.yaml Step 4 — Disabling Telemetry Step 5 — Usage & Shortcuts Troubleshooting What Are Continue and Ollama? Continue is an open-source VSCode (and JetBrains) extension that adds AI-powered coding features directly into your editor — think chat, inline edits, and tab autocomplete. By default it can connect to Claude, GPT-4, or any API. But it also supports local providers , which is exactly what we want here. Ollama i...

Local LLM on Ubuntu (CPU-Only): Ollama + Qwen Coder + Open WebUI + RAG Setup

Local LLM on Ubuntu (CPU-Only): Ollama + Qwen Coder + Open WebUI + RAG Run a private, high-performance internal AI coding assistant on your own hardware—no GPU required. Feb 13, 2026 Qwen 2.5 Coder RAG Enabled Enterprise Ready Keeping your code and documents on your own infrastructure is the gold standard for enterprise security. By combining Ollama , Qwen Coder , and Open WebUI , we can build a production-grade AI assistant that respects your privacy. Environment: CPU-only Ubuntu server (e.g., i9/Threadripper) with 64GB+ RAM for smooth 14B model inference. 1. System Preparation First, update your local environment and install necessary dependencies: sudo apt update && sudo apt upgrade -y sudo apt install -y curl git ca-certificates lscpu 2. Ollama & Model Tuning Install Ollama and optimize the configuration for CPU-only inference by setting the thread count: ...

Defining tomcat and mysql server Amazon EC2 instance

-- Create new amazon ec2 instance http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html -- Give permisson to pem file chmod 400 mycertificate.pem -- Connect to EC2 instance ssh -i mycertificate.pem ubuntu@ec2-51-51-51-51.eu-west-1.compute.amazonaws.com -- Installation Oracle Java 1.8 $ wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz $ tar -xzvf jdk-8u92-linux-x64.tar.gz $ sudo mkdir -p /usr/java/ $ sudo mv jdk1.8.0_92/ /usr/java/ $ sudo vim  /etc/profile         JAVA_HOME=/usr/java/jdk1.8.0_92         PATH=$PATH:$HOME/bin:$JAVA_HOME/bin         JRE_HOME=/usr/java/jdk1.8.0_92/jre         PATH=$PATH:$HOME/bin:$JRE_HOME/bin         export JAVA_HOME...