Home | About | Sematext search-lucene.com search-hadoop.com
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB
 Search Hadoop and all its subprojects:

Switch to Threaded View
MapReduce >> mail # user >> libjars error


Copy link to this message
-
Re: libjars error
You're not using the configuration provided by the ToolRunner instantiation:

Change:

Job job = new Job();

To:

Job job = new Job(getConf());

On Wed, Aug 28, 2013 at 12:09 AM, jamal sasha <[EMAIL PROTECTED]> wrote:
> I have bunch of jars whcih i want to pass. I am using libjars option to do
> so. But to do that I have to implement tool ??
> So i change my code to following but still I am getting this warning?
> 13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
> parsing the arguments. Applications should implement Tool for the same.
>
> and it is not finding that jar (in bold??)
> package org.foobar;
>
> import java.io.ByteArrayInputStream;
> import java.io.IOException;
> import java.io.InputStream;
> import java.util.Date;
>
> import org.apache.hadoop.conf.Configuration;
> import org.apache.hadoop.conf.Configured;
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.NullWritable;
> import org.apache.hadoop.io.Text;
> import com.hp.hpl.jena.rdf.model.*;
>
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.Mapper;
>
>
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
> import org.apache.hadoop.util.Tool;
> import org.apache.hadoop.util.ToolRunner;
>
>
> public class RdfFormatter extends Configured implements Tool {
>   //foo bar
> }
> public int run(String [] args) throws Exception{
> //Configuration conf = new Configuration();
>
>       Job job = new Job();
>       //job.setMapOutputKeyClass(Pair.class);
>       job.setOutputKeyClass(Text.class);
>       job.setOutputValueClass(Text.class);
>       //conf.set("delimiter", "\\s+");
>
>       job.setJarByClass(RdfFormatter.class);
>       job.setMapperClass(RDFMapper.class);
>       //job.setReducerClass(MeanReducer.class);
>       job.setNumReduceTasks(0);
>
>       job.setInputFormatClass(TextInputFormat.class);
>       job.setOutputFormatClass(TextOutputFormat.class);
>
>       FileInputFormat.addInputPath(job, new
> Path("/user/user/mohit/wiki-rdf/sample-input"));
>       FileOutputFormat.setOutputPath(job, new
> Path("/user/user/mohit/wiki-rdf/outputter"));
>       job.submit();
>       long start = new Date().getTime();
>       job.waitForCompletion(true);
>       long end = new Date().getTime();
>       System.out.println("Job took " + ((end-start)/1000) + " seconds" );
> return 0;
> }
> public static void main(String[] args) throws Exception {
> // TODO Auto-generated method stub
> /*
> Configuration conf = new Configuration();
>
>       Job job = new Job(conf, "RDF input format");
>       //job.setMapOutputKeyClass(Pair.class);
>       job.setOutputKeyClass(Text.class);
>       job.setOutputValueClass(IntWritable.class);
>       conf.set("delimiter", "\\s+");
>
>       job.setJarByClass(Formatter.class);
>       job.setMapperClass(Mapper.class);
>       //job.setReducerClass(Reducer.class);
>       job.setNumReduceTasks(0);
>
>       job.setInputFormatClass(TextInputFormat.class);
>       job.setOutputFormatClass(TextOutputFormat.class);
>
>       FileInputFormat.addInputPath(job, new Path("/input/data"));
>       FileOutputFormat.setOutputPath(job, new Path("/output"));
>       // set timer
>        *
>        */
> Configuration conf = new Configuration();
> int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
> run() method.
> System.exit(ret);
>
>
>
>
>
> }
>
> }
>

--
Harsh J
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB