Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
MapReduce >> mail # user >> muti-thread mapreduce

Copy link to this message
muti-thread mapreduce

I suddenly got this idea to do mapreduce job in a muti-thread way.
I don't know if it can work. Could anyone give me some advices?
Here is the java code:
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

public class LogProcessApp extends Thread {

 private static String sep;
 private String x2;
 private String x3;

 public LogProcessApp(String arg1,String arg2,String arg3){

 public static class CM extends Mapper<LongWritable, Text, Text, Text>{
  private Text keyvar = new Text();
  private Text valvar = new Text();
  public void map(LongWritable key, Text value, Context context)
    throws IOException, InterruptedException {
   String line = value.toString();
    String data[] = line.split(sep);
   } catch (Exception e) {

 public void run(){
  Configuration conf = new Configuration();
  Job job = null;
  try {
   job = new Job(conf);
  } catch (IOException e1) {
   // TODO Auto-generated catch block





  try {
   FileInputFormat.addInputPath(job, new Path(x2));
  } catch (IOException e) {
   // TODO Auto-generated catch block
  FileOutputFormat.setOutputPath(job, new Path(x3));

  try {
  } catch (IOException e) {
   // TODO Auto-generated catch block
  } catch (InterruptedException e) {
   // TODO Auto-generated catch block
  } catch (ClassNotFoundException e) {
   // TODO Auto-generated catch block

 public static void main(String args[]){
  LogProcessApp lpa1=new LogProcessApp(args[0],args[1],args[3]);
  LogProcessApp lpa2=new LogProcessApp(args[4],args[5],args[6]);
  LogProcessApp lpa3=new LogProcessApp(args[7],args[8],args[9]);