Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
MapReduce >> mail # user >> libjars error


Copy link to this message
-
Re: libjars error
You're not using the configuration provided by the ToolRunner instantiation:

Change:

Job job = new Job();

To:

Job job = new Job(getConf());

On Wed, Aug 28, 2013 at 12:09 AM, jamal sasha <[EMAIL PROTECTED]> wrote:
> I have bunch of jars whcih i want to pass. I am using libjars option to do
> so. But to do that I have to implement tool ??
> So i change my code to following but still I am getting this warning?
> 13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
>
> and it is not finding that jar (in bold??)
> package org.foobar;
>
> import java.io.ByteArrayInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.NullWritable;
> import org.apache.hadoop.io.Text;
> import com.hp.hpl.jena.rdf.model.*;
>
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
>
>
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class RdfFormatter extends Configured implements Tool {
>   //foo bar
> }
> public int run(String [] args) throws Exception{
> //Configuration conf = new Configuration();
>
>       Job job = new Job();
>       //job.setMapOutputKeyClass(Pair.class);
>       job.setOutputKeyClass(Text.class);
>       job.setOutputValueClass(Text.class);
>       //conf.set("delimiter", "\\s+");
>
>       job.setJarByClass(RdfFormatter.class);
>       job.setMapperClass(RDFMapper.class);
>       //job.setReducerClass(MeanReducer.class);
>       job.setNumReduceTasks(0);
>
>       job.setInputFormatClass(TextInputFormat.class);
>       job.setOutputFormatClass(TextOutputFormat.class);
>
>       FileInputFormat.addInputPath(job, new
> Path("/user/user/mohit/wiki-rdf/sample-input"));
>       FileOutputFormat.setOutputPath(job, new
> Path("/user/user/mohit/wiki-rdf/outputter"));
>       job.submit();
>       long start = new Date().getTime();
>       job.waitForCompletion(true);
>       long end = new Date().getTime();
>       System.out.println("Job took " + ((end-start)/1000) + " seconds" );
> return 0;
> }
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> /*
> Configuration conf = new Configuration();
>
>       Job job = new Job(conf, "RDF input format");
>       //job.setMapOutputKeyClass(Pair.class);
>       job.setOutputKeyClass(Text.class);
>       job.setOutputValueClass(IntWritable.class);
>       conf.set("delimiter", "\\s+");
>
>       job.setJarByClass(Formatter.class);
>       job.setMapperClass(Mapper.class);
>       //job.setReducerClass(Reducer.class);
>       job.setNumReduceTasks(0);
>
>       job.setInputFormatClass(TextInputFormat.class);
>       job.setOutputFormatClass(TextOutputFormat.class);
>
>       FileInputFormat.addInputPath(job, new Path("/input/data"));
>       FileOutputFormat.setOutputPath(job, new Path("/output"));
>       // set timer
>        *
>        */
> Configuration conf = new Configuration();
> int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
> run() method.
> System.exit(ret);
>
>
>
>
>
> }
>
> }
>

--
Harsh J